diff --git a/.github/workflows/pipeline-step.yml b/.github/workflows/pipeline-step.yml index dc7ceed..2c86652 100644 --- a/.github/workflows/pipeline-step.yml +++ b/.github/workflows/pipeline-step.yml @@ -46,7 +46,8 @@ jobs: if: inputs.runs-on == 'windows-2022' - run: synapse --version - - run: sudo apt-get update -y && sudo apt-get install ${{ fromJSON(inputs.step-config).systemDeps }} -y + - name: System Dependencies + run: sudo apt-get update -y && sudo apt-get install ${{ fromJSON(inputs.step-config).systemDeps }} -y if: fromJSON(inputs.step-config).systemDeps && startsWith(inputs.runs-on, 'ubuntu') - run: ${{ fromJSON(inputs.step-config).commands }} diff --git a/package.json b/package.json index a574f61..d0c20cd 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "zig": "0.13.0" }, "pipeline": { - "test": "synapse build && SYNAPSE_CMD=$(pwd)/dist/bin/synapse synapse run src/testing/internal.ts" + "test": "synapse build && SYNAPSE_CMD=$(pwd)/dist/bin/synapse synapse run src/testing/internal.ts && synapse clean && ./dist/bin/synapse compile" } } } \ No newline at end of file diff --git a/src/bundler.ts b/src/bundler.ts index 1eece96..3d70515 100644 --- a/src/bundler.ts +++ b/src/bundler.ts @@ -16,7 +16,7 @@ import { createModuleResolverForBundling } from './runtime/rootLoader' import { getWorkingDir } from './workspaces' import { pointerPrefix, createPointer, isDataPointer, toAbsolute, DataPointer, coerceToPointer, isNullHash, applyPointers } from './build-fs/pointers' import { getModuleType } from './static-solver' -import { readKeySync } from './cli/config' +import { readPathKeySync } from './cli/config' import { isSelfSea } from './execution' // Note: `//!` or `/*!` are considered "legal comments" @@ -71,7 +71,7 @@ export function createProgram( } export const setupEsbuild = memoize(() => { - const esbuildPath = readKeySync('esbuild.path') + const esbuildPath = readPathKeySync('esbuild.path') if (!esbuildPath) { if (!process.env.ESBUILD_BINARY_PATH && isSelfSea()) { throw new Error(`Missing esbuild binary`) diff --git a/src/cli/buildInternal.ts b/src/cli/buildInternal.ts index 4c462ce..98e9ba5 100644 --- a/src/cli/buildInternal.ts +++ b/src/cli/buildInternal.ts @@ -19,6 +19,7 @@ import { getLogger } from '../logging' import { randomUUID } from 'node:crypto' import { createZipFromDir } from '../deploy/deployment' import { tmpdir } from 'node:os' +import { readFileWithStats } from '../system' const integrations = { @@ -665,12 +666,17 @@ function stripComments(text: string) { } export async function createSynapseTarball(dir: string) { - const files = await glob(getFs(), dir, ['**/*', '**/.synapse']) - const tarball = createTarball(await Promise.all(files.map(async f => ({ - contents: Buffer.from(await getFs().readFile(f)), - mode: 0o755, - path: path.relative(dir, f), - })))) + const files = await glob(getFs(), dir, ['**/*', '**/.synapse']) + const tarball = createTarball(await Promise.all(files.map(async f => { + const { data, stats } = await readFileWithStats(f) + + return { + contents: Buffer.from(data), + mode: 0o755, + path: path.relative(dir, f), + mtime: Math.round(stats.mtimeMs), + } + }))) const zipped = await gzip(tarball) diff --git a/src/cli/config.ts b/src/cli/config.ts index 15fea9d..198ddcc 100644 --- a/src/cli/config.ts +++ b/src/cli/config.ts @@ -1,10 +1,11 @@ +import * as path from 'node:path' import { getFs } from '../execution' -import { tryReadJson, tryReadJsonSync } from '../utils' +import { memoize, tryReadJson, tryReadJsonSync, makeRelative } from '../utils' import { getUserConfigFilePath } from '../workspaces' let config: Record -let pendingConfig: Promise> -function _readConfig(): Promise> | Record { +let pendingConfig: Promise> | undefined +function readConfig(): Promise> | Record { if (config) { return config } @@ -14,13 +15,22 @@ function _readConfig(): Promise> | Record { } return pendingConfig = tryReadJson(getFs(), getUserConfigFilePath()).then(val => { - val ??= {} - return config = val as any + return config = (val ?? {}) as any + }).finally(() => { + pendingConfig = undefined }) } +function readConfigSync(): Record { + if (config) { + return config + } + + return config = (tryReadJsonSync(getFs(), getUserConfigFilePath()) ?? {}) +} + let pendingWrite: Promise | undefined -function _writeConfig(conf: Record) { +function writeConfig(conf: Record) { config = conf const write = () => getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4)) @@ -43,19 +53,8 @@ function _writeConfig(conf: Record) { } pendingWrite = undefined }) - return pendingWrite = tmp -} - -async function readConfig(): Promise> { - return (await tryReadJson(getFs(), getUserConfigFilePath())) ?? {} -} - -async function writeConfig(conf: Record): Promise { - await getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4)) -} -function readConfigSync(): Record { - return tryReadJsonSync(getFs(), getUserConfigFilePath()) ?? {} + return pendingWrite = tmp } function getValue(val: any, key: string) { @@ -98,5 +97,40 @@ export async function setKey(key: string, value: any) { return oldValue } +const getUserConfigDir = memoize(() => path.dirname(getUserConfigFilePath())) + +export function readPathKeySync(key: string): string | undefined { + const val = readKeySync(key) + + return val !== undefined ? path.resolve(getUserConfigDir(), val) : val +} + +export async function readPathKey(key: string): Promise { + const val = await readKey(key) + + return val !== undefined ? path.resolve(getUserConfigDir(), val) : val +} + +export async function readPathMapKey(key: string): Promise | undefined> { + const val = await readKey>(key) + if (!val) { + return + } + + const resolved: Record = {} + for (const k of Object.keys(val)) { + resolved[k] = path.resolve(getUserConfigDir(), val[k]) + } + + return resolved +} + +export async function setPathKey(key: string, value: string) { + const abs = path.resolve(getUserConfigDir(), value) + const rel = makeRelative(getUserConfigDir(), abs) + + return setKey(key, rel) +} + // synapse.cli.suggestions -> false // Need settings to change where test/deploy/synth logs go diff --git a/src/cli/index.ts b/src/cli/index.ts index d7abf0b..b27bffe 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -5,8 +5,8 @@ import * as path from 'node:path' import * as inspector from 'node:inspector' import { getLogger } from '../logging' import { LogLevel, logToFile, logToStderr, purgeOldLogs, validateLogLevel } from './logger' -import { CancelError, getCurrentVersion, runWithContext, setContext, setCurrentVersion } from '../execution' -import { RenderableError, colorize, getDisplay, printJson, printLine } from './ui' +import { CancelError, dispose, getCurrentVersion, runWithContext, setContext, setCurrentVersion } from '../execution' +import { RenderableError, colorize, getDisplay, printLine } from './ui' import { showUsage, executeCommand, runWithAnalytics, removeInternalCommands } from './commands' import { getCiType } from '../utils' import { resolveProgramBuildTarget } from '../workspaces' @@ -288,7 +288,7 @@ export function main(...args: string[]) { didThrow = true } finally { - await getDisplay().dispose() + await dispose() await disposable?.dispose() // No more log events will be emitted setTimeout(() => { diff --git a/src/cli/ui.ts b/src/cli/ui.ts index f2d7f2d..fc813b8 100644 --- a/src/cli/ui.ts +++ b/src/cli/ui.ts @@ -2,6 +2,7 @@ import { getLogger } from '../logging' import { createEventEmitter } from '../events' import { memoize } from '../utils' import * as nodeUtil from 'node:util' +import { pushDisposable } from '../execution' const esc = '\x1b' @@ -115,7 +116,13 @@ export function stripAnsi(s: string) { let display: ReturnType export function getDisplay() { - return display ??= createDisplay() + if (display) { + return display + } + + display = createDisplay() + + return pushDisposable(display) } function swap(arr: any[], i: number, j: number) { @@ -1272,7 +1279,13 @@ export function createDisplay() { writer.tty?.dispose() } + let disposed = false async function dispose() { + if (disposed) { + return + } + + disposed = true if (!process.stdout.isTTY) { return } @@ -1280,7 +1293,14 @@ export function createDisplay() { await releaseTty(true) } - return { createView, getOverlayedView, dispose, releaseTty, writer } + return { + createView, + getOverlayedView, + dispose, + releaseTty, + writer, + [Symbol.asyncDispose]: dispose, + } } export interface TreeItem { diff --git a/src/compiler/config.ts b/src/compiler/config.ts index fcd8eae..9535546 100644 --- a/src/compiler/config.ts +++ b/src/compiler/config.ts @@ -9,7 +9,7 @@ import { PackageJson, getPreviousPkg } from '../pm/packageJson' import { getProgramFs } from '../artifacts' import { getWorkingDir } from '../workspaces' import { getHash, isWindows, makeRelative, memoize, resolveRelative, throwIfNotFileNotFoundError } from '../utils' -import { readKeySync } from '../cli/config' +import { readPathKeySync } from '../cli/config' interface ParsedConfig { readonly cmd: Pick @@ -692,7 +692,7 @@ function libFromTarget(target: ts.ScriptTarget) { const patchTsSys = memoize(() => { // The filepath returned by `getExecutingFilePath` doesn't need to exist - const libDir = readKeySync('typescript.libDir') + const libDir = readPathKeySync('typescript.libDir') if (typeof libDir === 'string') { ts.sys.getExecutingFilePath = () => path.resolve(libDir, 'cli.js') diff --git a/src/compiler/incremental.ts b/src/compiler/incremental.ts index 21e707b..b9f2933 100644 --- a/src/compiler/incremental.ts +++ b/src/compiler/incremental.ts @@ -3,7 +3,7 @@ import * as path from 'node:path' import { createFileHasher, isWindows, keyedMemoize, memoize, throwIfNotFileNotFoundError } from '../utils' import { Fs } from '../system' import { getGlobalCacheDirectory } from '../workspaces' -import { getFs } from '../execution' +import { getFs, pushDisposable } from '../execution' import { getProgramFs } from '../artifacts' interface DependencyEdge { @@ -183,7 +183,11 @@ export async function clearIncrementalCache() { await getProgramFs().deleteFile(incrementalFileName).catch(throwIfNotFileNotFoundError) } -export const getFileHasher = memoize(() => createFileHasher(getFs(), getGlobalCacheDirectory())) +export const getFileHasher = memoize(() => { + const hasher = createFileHasher(getFs(), getGlobalCacheDirectory()) + + return pushDisposable(hasher) +}) // TODO: clear cache when updating packages export type IncrementalHost = ReturnType @@ -251,10 +255,7 @@ export function createIncrementalHost(opt: ts.CompilerOptions) { }) } - await Promise.all([ - saveCache({ ...cache, ...updatedCache }), - fileChecker.flush(), - ]) + await saveCache({ ...cache, ...updatedCache }) if (isWindows()) { return new Set([...changed].map(f => f.replaceAll('\\', '/'))) diff --git a/src/deploy/deployment.ts b/src/deploy/deployment.ts index c777d62..caa5f8b 100644 --- a/src/deploy/deployment.ts +++ b/src/deploy/deployment.ts @@ -14,7 +14,7 @@ import { TfState } from './state' import { randomUUID } from 'node:crypto' import { getFsFromHash, getDeploymentFs, getProgramFs, getProgramHash, getResourceProgramHashes, getTemplate, putState, setResourceProgramHashes } from '../artifacts' import { runCommand } from '../utils/process' -import { readKey } from '../cli/config' +import { readPathKey } from '../cli/config' import { getDisplay, spinners } from '../cli/ui' import { readDirRecursive } from '../system' @@ -1226,7 +1226,7 @@ export function renderPlan(plan: TfPlan) { export async function getTerraformPath() { // This is configured on installation. - const configuredPath = await readKey('terraform.path') + const configuredPath = await readPathKey('terraform.path') if (configuredPath) { return configuredPath } diff --git a/src/execution.ts b/src/execution.ts index 9a2ecdc..40b0710 100644 --- a/src/execution.ts +++ b/src/execution.ts @@ -109,6 +109,26 @@ export function getSelfPathOrThrow() { export class CancelError extends Error {} +// These are global for now +const disposables: (Disposable | AsyncDisposable)[] = [] +export function pushDisposable(disposable: T): T { + disposables.push(disposable) + return disposable +} + +export async function dispose() { + const promises: PromiseLike[] = [] + for (const d of disposables) { + if (Symbol.dispose in d) { + d[Symbol.dispose]() + } else { + promises.push(d[Symbol.asyncDispose]()) + } + } + + disposables.length = 0 + await Promise.all(promises) +} // This is mutable and may be set at build-time let semver = '0.0.1' diff --git a/src/index.ts b/src/index.ts index 452fdca..cc6f181 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,7 +18,7 @@ import { createTemplateService, getHash, parseModuleName } from './templates' import { createImportMap, createModuleResolver } from './runtime/resolver' import { createAuth, getAuth } from './auth' import { generateOpenApiV3, generateStripeWebhooks } from './codegen/schemas' -import { createNpmLikeCommandRunner, dumpPackage, emitPackageDist, getPkgExecutables, getProjectOverridesMapping, installToUserPath, linkPackage, publishToRemote } from './pm/publish' +import { createNpmLikeCommandRunner, dumpPackage, emitPackageDist, getPkgExecutables, getProjectOverridesMapping, linkPackage, publishToRemote } from './pm/publish' import { ResolvedProgramConfig, getResolvedTsConfig, resolveProgramConfig } from './compiler/config' import { createProgramBuilder, getDeployables, getEntrypointsFile, getExecutables } from './compiler/programBuilder' import { loadCpuProfile } from './perf/profiles' @@ -1757,6 +1757,7 @@ export async function startWatch(targets?: string[], opt?: CompilerOptions & { a } } } + getLogger().debug(`Changed infra files:`, [...changedDeployables]) if (changedDeployables.size > 0 && config.csc.deployTarget) { @@ -1778,6 +1779,8 @@ export async function startWatch(targets?: string[], opt?: CompilerOptions & { a // XXX await afs.clearCurrentProgramStore() } + + await getFileHasher().flush() } const afterProgramCreate = watchHost.afterProgramCreate diff --git a/src/pm/publish.ts b/src/pm/publish.ts index 91ffc35..046d358 100644 --- a/src/pm/publish.ts +++ b/src/pm/publish.ts @@ -12,7 +12,7 @@ import { getBuildTargetOrThrow, getFs, getSelfPathOrThrow, isSelfSea } from '../ import { ImportMap, expandImportMap, hoistImportMap } from '../runtime/importMaps' import { createCommandRunner, patchPath, runCommand } from '../utils/process' import { PackageJson, ResolvedPackage, getCompiledPkgJson, getCurrentPkg, getImmediatePackageJsonOrThrow, getPackageJson } from './packageJson' -import { readKey, setKey } from '../cli/config' +import { readPathMapKey, setPathKey } from '../cli/config' import { getEntrypointsFile } from '../compiler/programBuilder' import { createPackageForRelease, createSynapseTarball } from '../cli/buildInternal' import * as registry from '@cohesible/resources/registry' @@ -228,9 +228,7 @@ export async function linkPackage(opt?: PublishOptions & { globalInstall?: boole } async function replaceIntegration(name: string) { - const overrides = await readKey>('projectOverrides') ?? {} - overrides[`synapse-${name}`] = resolvedDir - await setKey('projectOverrides', overrides) + await setPathKey(`projectOverrides.synapse-${name}`, resolvedDir) } if (pkgName.startsWith('synapse-')) { @@ -337,7 +335,7 @@ async function getOverridesFromProject() { async function getMergedOverrides() { const [fromConfig, fromProject] = await Promise.all([ - readKey>('projectOverrides'), + readPathMapKey('projectOverrides'), getOverridesFromProject() ]) diff --git a/src/runtime/modules/http.ts b/src/runtime/modules/http.ts index 0867445..ac44228 100644 --- a/src/runtime/modules/http.ts +++ b/src/runtime/modules/http.ts @@ -342,6 +342,12 @@ function resolveBody(body: any) { } } + if (body instanceof ArrayBuffer) { + return { + body: typeof Buffer !== 'undefined' ? Buffer.from(body) : body, + } + } + if (isTypedArray(body)) { const contentEncoding = body[contentEncodingSym] @@ -741,7 +747,12 @@ function doRequest(request: http.RequestOptions | URL, body?: any) { } if (isJson && result) { - const e = JSON.parse(decoded.toString('utf-8')) + let e: any | undefined + try { + e = JSON.parse(decoded.toString('utf-8')) + } catch (parseErr) { + e = new Error(decoded.toString('utf-8')) + } err.message = e.message ?? `Received non-2xx status code: ${res.statusCode}` const stack = `${e.name ?? 'Error'}: ${err.message}\n` + err.stack?.split('\n').slice(1).join('\n') diff --git a/src/runtime/srl/compute/index.ts b/src/runtime/srl/compute/index.ts index 4208009..19ac36a 100644 --- a/src/runtime/srl/compute/index.ts +++ b/src/runtime/srl/compute/index.ts @@ -169,3 +169,21 @@ export declare class Website { export declare class Schedule { public constructor(expression: string, fn: () => Promise | void) } + +type PromisifyFunction = T extends (...args: infer A) => Promise + ? T : T extends (...args: infer A) => infer U + ? (...args: A) => Promise : T + +type Promisify = { [P in keyof T]: PromisifyFunction } + +//# resource = true +/** + * @internal Used internally for registering environment-agnostic API handlers + */ +export declare class SecureService { + public constructor(obj: T) + + public serialize(): string & { __type?: T } + + public static deserialize(encoded: string & { __type?: T }): Promisify +} diff --git a/src/utils.ts b/src/utils.ts index a53bcb3..e2544c8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1610,7 +1610,7 @@ interface FileHasherCacheWithTime { // This is should ideally only be used for source files export function createFileHasher(fs: Pick, cacheLocation: string) { // We use a single timestamp to represent the entire session - const checkTime = Date.now() + let checkTime = Date.now() const location = path.resolve(cacheLocation, 'files.json') async function loadCache(): Promise { @@ -1664,15 +1664,18 @@ export function createFileHasher(fs: Pick { - delete (await cachePromise)[fileName] + delete cache[fileName] throw e }) // TODO: try rounding `mtime`? - const cache = await cachePromise - const cached = cache[fileName] if (cached && cached.mtime === stat.mtimeMs) { cached.checkTime = checkTime @@ -1687,15 +1690,20 @@ export function createFileHasher(fs: Pick= 100) { @@ -119,8 +120,7 @@ export function createTarball(files: TarballFile[]): Buffer { buf.write(toOctal(uid, 8), i + 108, 8, 'ascii') buf.write(toOctal(gid, 8), i + 116, 8, 'ascii') buf.write(toOctal(size, 12), i + 124, 12, 'ascii') - buf.write(toOctal(mtime, 12), i + 136, 12, 'ascii') - + buf.write(toOctal(f.mtime, 12), i + 136, 12, 'ascii') buf.write(' '.repeat(8), i + 148, 8, 'ascii') let checksum = 0 diff --git a/src/zig/installer.ts b/src/zig/installer.ts index 1fc929a..c5ce6bc 100644 --- a/src/zig/installer.ts +++ b/src/zig/installer.ts @@ -5,7 +5,7 @@ import { fetchData, fetchJson } from '../utils/http' import { getPackageCacheDirectory, getUserSynapseDirectory, getWorkingDir } from '../workspaces' import { runCommand } from '../utils/process' import { ensureDir, isNonNullable, isWindows, memoize } from '../utils' -import { readKey, setKey } from '../cli/config' +import { readPathKey, setPathKey } from '../cli/config' import { extractToDir } from '../utils/tar' import { registerToolProvider } from '../pm/tools' import { getFs } from '../execution' @@ -209,7 +209,7 @@ export async function getZigPath() { return fromNodeModules } - const fromConfig = await readKey('zig.path') + const fromConfig = await readPathKey('zig.path') if (fromConfig) { return fromConfig } @@ -223,13 +223,13 @@ export async function getZigPath() { throw new Error(`Unexpected zig version install: found ${v}, expected: ${expected}`) } - await setKey('zig.path', zigPath) + await setPathKey('zig.path', zigPath) return zigPath } function getSynapseZigLibDir() { - return readKey('zig.synapseLibDir') + return readPathKey('zig.synapseLibDir') } export async function getJsLibPath() {