Test

.github/workflows/pipeline-step.yml

@@ -46,7 +46,8 @@ jobs:
         if: inputs.runs-on == 'windows-2022'
       - run: synapse --version
 
-      - run: sudo apt-get update -y && sudo apt-get install ${{ fromJSON(inputs.step-config).systemDeps }} -y
+      - name: System Dependencies
+        run: sudo apt-get update -y && sudo apt-get install ${{ fromJSON(inputs.step-config).systemDeps }} -y
         if: fromJSON(inputs.step-config).systemDeps && startsWith(inputs.runs-on, 'ubuntu')
 
       - run: ${{ fromJSON(inputs.step-config).commands }}

package.json

@@ -33,7 +33,7 @@
             "zig": "0.13.0"
         },
         "pipeline": {
-            "test": "synapse build && SYNAPSE_CMD=$(pwd)/dist/bin/synapse synapse run src/testing/internal.ts"
+            "test": "synapse build && SYNAPSE_CMD=$(pwd)/dist/bin/synapse synapse run src/testing/internal.ts && synapse clean && ./dist/bin/synapse compile"
         }
     }
 }

src/bundler.ts

@@ -16,7 +16,7 @@ import { createModuleResolverForBundling } from './runtime/rootLoader'
 import { getWorkingDir } from './workspaces'
 import { pointerPrefix, createPointer, isDataPointer, toAbsolute, DataPointer, coerceToPointer, isNullHash, applyPointers } from './build-fs/pointers'
 import { getModuleType } from './static-solver'
-import { readKeySync } from './cli/config'
+import { readPathKeySync } from './cli/config'
 import { isSelfSea } from './execution'
 
 // Note: `//!` or `/*!` are considered "legal comments"
@@ -71,7 +71,7 @@ export function createProgram(
 }
 
 export const setupEsbuild = memoize(() => {
-    const esbuildPath = readKeySync<string>('esbuild.path')
+    const esbuildPath = readPathKeySync('esbuild.path')
     if (!esbuildPath) {
         if (!process.env.ESBUILD_BINARY_PATH && isSelfSea()) {
             throw new Error(`Missing esbuild binary`)

src/cli/buildInternal.ts

@@ -19,6 +19,7 @@ import { getLogger } from '../logging'
 import { randomUUID } from 'node:crypto'
 import { createZipFromDir } from '../deploy/deployment'
 import { tmpdir } from 'node:os'
+import { readFileWithStats } from '../system'
 
 
 const integrations = {
@@ -665,12 +666,17 @@ function stripComments(text: string) {
 }
 
 export async function createSynapseTarball(dir: string) {
-    const files = await glob(getFs(), dir, ['**/*', '**/.synapse'])    
-    const tarball = createTarball(await Promise.all(files.map(async f => ({
-        contents: Buffer.from(await getFs().readFile(f)),
-        mode: 0o755,
-        path: path.relative(dir, f),
-    }))))
+    const files = await glob(getFs(), dir, ['**/*', '**/.synapse'])
+    const tarball = createTarball(await Promise.all(files.map(async f => {
+        const { data, stats } = await readFileWithStats(f)
+
+        return {
+            contents: Buffer.from(data),
+            mode: 0o755,
+            path: path.relative(dir, f),
+            mtime: Math.round(stats.mtimeMs),
+        }
+    })))
 
     const zipped = await gzip(tarball)
 

src/cli/config.ts

@@ -1,10 +1,11 @@
+import * as path from 'node:path'
 import { getFs } from '../execution'
-import { tryReadJson, tryReadJsonSync } from '../utils'
+import { memoize, tryReadJson, tryReadJsonSync, makeRelative } from '../utils'
 import { getUserConfigFilePath } from '../workspaces'
 
 let config: Record<string, any>
-let pendingConfig: Promise<Record<string, any>>
-function _readConfig(): Promise<Record<string, any>> | Record<string, any> {
+let pendingConfig: Promise<Record<string, any>> | undefined
+function readConfig(): Promise<Record<string, any>> | Record<string, any> {
     if (config) {
         return config
     }
@@ -14,13 +15,22 @@ function _readConfig(): Promise<Record<string, any>> | Record<string, any> {
     }
 
     return pendingConfig = tryReadJson(getFs(), getUserConfigFilePath()).then(val => {
-        val ??= {}
-        return config = val as any
+        return config = (val ?? {}) as any
+    }).finally(() => {
+        pendingConfig = undefined
     })
 }
 
+function readConfigSync(): Record<string, any> {
+    if (config) {
+        return config
+    }
+
+    return config = (tryReadJsonSync(getFs(), getUserConfigFilePath()) ?? {})
+}
+
 let pendingWrite: Promise<void> | undefined
-function _writeConfig(conf: Record<string, any>) {
+function writeConfig(conf: Record<string, any>) {
     config = conf
 
     const write = () => getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4))
@@ -43,19 +53,8 @@ function _writeConfig(conf: Record<string, any>) {
         }
         pendingWrite = undefined
     })
-    return pendingWrite = tmp
-}
-
-async function readConfig(): Promise<Record<string, any>> {
-    return (await tryReadJson(getFs(), getUserConfigFilePath())) ?? {}
-}
-
-async function writeConfig(conf: Record<string, any>): Promise<void> {
-    await getFs().writeFile(getUserConfigFilePath(), JSON.stringify(conf, undefined, 4))
-}
 
-function readConfigSync(): Record<string, any> {
-    return tryReadJsonSync(getFs(), getUserConfigFilePath()) ?? {}
+    return pendingWrite = tmp
 }
 
 function getValue(val: any, key: string) {
@@ -98,5 +97,40 @@ export async function setKey(key: string, value: any) {
     return oldValue
 }
 
+const getUserConfigDir = memoize(() => path.dirname(getUserConfigFilePath()))
+
+export function readPathKeySync(key: string): string | undefined {
+    const val = readKeySync<string>(key)
+
+    return val !== undefined ? path.resolve(getUserConfigDir(), val) : val
+}
+
+export async function readPathKey(key: string): Promise<string | undefined> {
+    const val = await readKey<string>(key)
+
+    return val !== undefined ? path.resolve(getUserConfigDir(), val) : val
+}
+
+export async function readPathMapKey(key: string): Promise<Record<string, string> | undefined> {
+    const val = await readKey<Record<string, string>>(key)
+    if (!val) {
+        return
+    }
+
+    const resolved: Record<string, string> = {}
+    for (const k of Object.keys(val)) {
+        resolved[k] = path.resolve(getUserConfigDir(), val[k])
+    }
+
+    return resolved
+}
+
+export async function setPathKey(key: string, value: string) {
+    const abs = path.resolve(getUserConfigDir(), value)
+    const rel = makeRelative(getUserConfigDir(), abs)
+
+    return setKey(key, rel)
+}
+
 // synapse.cli.suggestions -> false
 // Need settings to change where test/deploy/synth logs go

src/cli/index.ts

@@ -5,8 +5,8 @@ import * as path from 'node:path'
 import * as inspector from 'node:inspector'
 import { getLogger } from '../logging'
 import { LogLevel, logToFile, logToStderr, purgeOldLogs, validateLogLevel } from './logger'
-import { CancelError, getCurrentVersion, runWithContext, setContext, setCurrentVersion } from '../execution'
-import { RenderableError, colorize, getDisplay, printJson, printLine } from './ui'
+import { CancelError, dispose, getCurrentVersion, runWithContext, setContext, setCurrentVersion } from '../execution'
+import { RenderableError, colorize, getDisplay, printLine } from './ui'
 import { showUsage, executeCommand, runWithAnalytics, removeInternalCommands } from './commands'
 import { getCiType } from '../utils'
 import { resolveProgramBuildTarget } from '../workspaces'
@@ -288,7 +288,7 @@ export function main(...args: string[]) {
 
             didThrow = true
         } finally {
-            await getDisplay().dispose()
+            await dispose()
             await disposable?.dispose() // No more log events will be emitted
 
             setTimeout(() => {

src/cli/ui.ts

@@ -2,6 +2,7 @@ import { getLogger } from '../logging'
 import { createEventEmitter } from '../events'
 import { memoize } from '../utils'
 import * as nodeUtil from 'node:util'
+import { pushDisposable } from '../execution'
 
 const esc = '\x1b'
 
@@ -115,7 +116,13 @@ export function stripAnsi(s: string) {
 
 let display: ReturnType<typeof createDisplay>
 export function getDisplay() {
-    return display ??= createDisplay()
+    if (display) {
+        return display
+    }
+
+    display = createDisplay()
+
+    return pushDisposable(display)
 }
 
 function swap(arr: any[], i: number, j: number) {
@@ -1272,15 +1279,28 @@ export function createDisplay() {
         writer.tty?.dispose()
     }
 
+    let disposed = false
     async function dispose() {
+        if (disposed) {
+            return
+        }
+
+        disposed = true
         if (!process.stdout.isTTY) {
             return
         }
 
         await releaseTty(true)
     }
 
-    return { createView, getOverlayedView, dispose, releaseTty, writer }
+    return { 
+        createView, 
+        getOverlayedView, 
+        dispose, 
+        releaseTty, 
+        writer,
+        [Symbol.asyncDispose]: dispose,
+    }
 }
 
 export interface TreeItem {

src/compiler/config.ts

@@ -9,7 +9,7 @@ import { PackageJson, getPreviousPkg } from '../pm/packageJson'
 import { getProgramFs } from '../artifacts'
 import { getWorkingDir } from '../workspaces'
 import { getHash, isWindows, makeRelative, memoize, resolveRelative, throwIfNotFileNotFoundError } from '../utils'
-import { readKeySync } from '../cli/config'
+import { readPathKeySync } from '../cli/config'
 
 interface ParsedConfig {
     readonly cmd: Pick<ts.ParsedCommandLine, 'options' | 'fileNames' | 'raw'>
@@ -692,7 +692,7 @@ function libFromTarget(target: ts.ScriptTarget) {
 
 const patchTsSys = memoize(() => {
     // The filepath returned by `getExecutingFilePath` doesn't need to exist
-    const libDir = readKeySync('typescript.libDir')
+    const libDir = readPathKeySync('typescript.libDir')
     if (typeof libDir === 'string') {
         ts.sys.getExecutingFilePath = () => path.resolve(libDir, 'cli.js')
 

src/compiler/incremental.ts

@@ -3,7 +3,7 @@ import * as path from 'node:path'
 import { createFileHasher, isWindows, keyedMemoize, memoize, throwIfNotFileNotFoundError } from '../utils'
 import { Fs } from '../system'
 import { getGlobalCacheDirectory } from '../workspaces'
-import { getFs } from '../execution'
+import { getFs, pushDisposable } from '../execution'
 import { getProgramFs } from '../artifacts'
 
 interface DependencyEdge {
@@ -183,7 +183,11 @@ export async function clearIncrementalCache() {
     await getProgramFs().deleteFile(incrementalFileName).catch(throwIfNotFileNotFoundError)
 }
 
-export const getFileHasher = memoize(() =>  createFileHasher(getFs(), getGlobalCacheDirectory()))
+export const getFileHasher = memoize(() => {
+    const hasher = createFileHasher(getFs(), getGlobalCacheDirectory())
+
+    return pushDisposable(hasher)
+})
 
 // TODO: clear cache when updating packages
 export type IncrementalHost = ReturnType<typeof createIncrementalHost>
@@ -251,10 +255,7 @@ export function createIncrementalHost(opt: ts.CompilerOptions) {
             })
         }
 
-        await Promise.all([
-            saveCache({ ...cache, ...updatedCache }),
-            fileChecker.flush(),
-        ])
+        await saveCache({ ...cache, ...updatedCache })
 
         if (isWindows()) {
             return new Set([...changed].map(f => f.replaceAll('\\', '/')))

src/deploy/deployment.ts

@@ -14,7 +14,7 @@ import { TfState } from './state'
 import { randomUUID } from 'node:crypto'
 import { getFsFromHash, getDeploymentFs, getProgramFs, getProgramHash, getResourceProgramHashes, getTemplate, putState, setResourceProgramHashes } from '../artifacts'
 import { runCommand } from '../utils/process'
-import { readKey } from '../cli/config'
+import { readPathKey } from '../cli/config'
 import { getDisplay, spinners } from '../cli/ui'
 import { readDirRecursive } from '../system'
 
@@ -1226,7 +1226,7 @@ export function renderPlan(plan: TfPlan) {
 
 export async function getTerraformPath() {
     // This is configured on installation.
-    const configuredPath = await readKey<string>('terraform.path')
+    const configuredPath = await readPathKey('terraform.path')
     if (configuredPath) {
         return configuredPath
     }

src/execution.ts

@@ -109,6 +109,26 @@ export function getSelfPathOrThrow() {
 
 export class CancelError extends Error {}
 
+// These are global for now
+const disposables: (Disposable | AsyncDisposable)[] = []
+export function pushDisposable<T extends Disposable | AsyncDisposable>(disposable: T): T {
+    disposables.push(disposable)
+    return disposable
+}
+
+export async function dispose() {
+    const promises: PromiseLike<void>[] = []
+    for (const d of disposables) {
+        if (Symbol.dispose in d) {
+            d[Symbol.dispose]()
+        } else {
+            promises.push(d[Symbol.asyncDispose]())
+        }
+    }
+
+    disposables.length = 0
+    await Promise.all(promises)
+}
 
 // This is mutable and may be set at build-time
 let semver = '0.0.1'

src/index.ts

@@ -18,7 +18,7 @@ import { createTemplateService, getHash, parseModuleName } from './templates'
 import { createImportMap, createModuleResolver } from './runtime/resolver'
 import { createAuth, getAuth } from './auth'
 import { generateOpenApiV3, generateStripeWebhooks } from './codegen/schemas'
-import { createNpmLikeCommandRunner, dumpPackage, emitPackageDist, getPkgExecutables, getProjectOverridesMapping, installToUserPath, linkPackage, publishToRemote } from './pm/publish'
+import { createNpmLikeCommandRunner, dumpPackage, emitPackageDist, getPkgExecutables, getProjectOverridesMapping, linkPackage, publishToRemote } from './pm/publish'
 import { ResolvedProgramConfig, getResolvedTsConfig, resolveProgramConfig } from './compiler/config'
 import { createProgramBuilder, getDeployables, getEntrypointsFile, getExecutables } from './compiler/programBuilder'
 import { loadCpuProfile } from './perf/profiles'
@@ -1757,6 +1757,7 @@ export async function startWatch(targets?: string[], opt?: CompilerOptions & { a
                 }
             }
         }
+
         getLogger().debug(`Changed infra files:`, [...changedDeployables])
 
         if (changedDeployables.size > 0 && config.csc.deployTarget) {
@@ -1778,6 +1779,8 @@ export async function startWatch(targets?: string[], opt?: CompilerOptions & { a
             // XXX
             await afs.clearCurrentProgramStore()
         }
+
+        await getFileHasher().flush()
     }
 
     const afterProgramCreate = watchHost.afterProgramCreate

src/pm/publish.ts

@@ -12,7 +12,7 @@ import { getBuildTargetOrThrow, getFs, getSelfPathOrThrow, isSelfSea } from '../
 import { ImportMap, expandImportMap, hoistImportMap } from '../runtime/importMaps'
 import { createCommandRunner, patchPath, runCommand } from '../utils/process'
 import { PackageJson, ResolvedPackage, getCompiledPkgJson, getCurrentPkg, getImmediatePackageJsonOrThrow, getPackageJson } from './packageJson'
-import { readKey, setKey } from '../cli/config'
+import { readPathMapKey, setPathKey } from '../cli/config'
 import { getEntrypointsFile } from '../compiler/programBuilder'
 import { createPackageForRelease, createSynapseTarball } from '../cli/buildInternal'
 import * as registry from '@cohesible/resources/registry'
@@ -228,9 +228,7 @@ export async function linkPackage(opt?: PublishOptions & { globalInstall?: boole
     }
 
     async function replaceIntegration(name: string) {
-        const overrides = await readKey<Record<string, string>>('projectOverrides') ?? {}
-        overrides[`synapse-${name}`] = resolvedDir
-        await setKey('projectOverrides', overrides)
+        await setPathKey(`projectOverrides.synapse-${name}`, resolvedDir)
     }
 
     if (pkgName.startsWith('synapse-')) {
@@ -337,7 +335,7 @@ async function getOverridesFromProject() {
 
 async function getMergedOverrides() {
     const [fromConfig, fromProject] = await Promise.all([
-        readKey<Record<string, string>>('projectOverrides'),
+        readPathMapKey('projectOverrides'),
         getOverridesFromProject()
     ])