Merge branch 'main' into toph/onramp

docs/lib/duckdb.md

@@ -140,7 +140,11 @@ SELECT ST_Area('POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))'::GEOMETRY) as area;
 
 To tell which extensions have been loaded, you can run the following query:
 
-```sql echo
+```sql
+FROM duckdb_extensions() WHERE loaded AND JSON '1';
+```
+
+```sql run=false
 FROM duckdb_extensions() WHERE loaded;
 ```
 

docs/lib/sqlite.md

@@ -6,6 +6,14 @@
 import SQLite from "npm:@observablehq/sqlite";
 ```
 
+If you prefer to use sql.js directly, you can import and initialize it like so:
+
+```js run=false
+import initSqlJs from "npm:sql.js";
+
+const SQLite = await initSqlJs({locateFile: (name) => import.meta.resolve("npm:sql.js/dist/") + name});
+```
+
 We also provide `SQLiteDatabaseClient`, a [`DatabaseClient`](https://observablehq.com/@observablehq/database-client-specification) implementation.
 
 ```js run=false

package.json

@@ -58,7 +58,7 @@
     "@clack/prompts": "^0.7.0",
     "@observablehq/inputs": "^0.12.0",
     "@observablehq/inspector": "^5.0.1",
-    "@observablehq/runtime": "^6.0.0-rc.1",
+    "@observablehq/runtime": "^6.0.0",
     "@rollup/plugin-commonjs": "^25.0.7",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",

src/build.ts

@@ -4,7 +4,7 @@ import {copyFile, readFile, rm, stat, writeFile} from "node:fs/promises";
 import {basename, dirname, extname, join} from "node:path/posix";
 import type {Config} from "./config.js";
 import {getDuckDBManifest} from "./duckdb.js";
-import {CliError} from "./error.js";
+import {CliError, enoent} from "./error.js";
 import {getClientPath, prepareOutput} from "./files.js";
 import {findModule, getModuleHash, readJavaScript} from "./javascript/module.js";
 import {transpileModule} from "./javascript/transpile.js";
@@ -54,7 +54,7 @@ export async function build(
   {config}: BuildOptions,
   effects: BuildEffects = new FileBuildEffects(config.output, join(config.root, ".observablehq", "cache"))
 ): Promise<void> {
-  const {root, loaders, duckdb} = config;
+  const {root, loaders, title, duckdb} = config;
   Telemetry.record({event: "build", step: "start"});
 
   // Prepare for build (such as by emptying the existing output root).
@@ -75,6 +75,25 @@ export async function build(
   let assetCount = 0;
   let pageCount = 0;
   const pagePaths = new Set<string>();
+
+  const buildManifest: BuildManifest = {
+    ...(title && {title}),
+    config: {root},
+    pages: [],
+    modules: [],
+    files: []
+  };
+
+  // file is the serving path relative to the base (e.g., /foo)
+  // path is the source file relative to the source root (e.g., /foo.md)
+  const addToManifest = (type: string, file: string, {title, path}: {title?: string | null; path: string}) => {
+    buildManifest[type].push({
+      path: config.normalizePath(file),
+      source: join("/", path), // TODO have route return path with leading slash?
+      ...(title != null && {title})
+    });
+  };
+
   for await (const path of config.paths()) {
     effects.output.write(`${faint("load")} ${path} `);
     const start = performance.now();
@@ -91,6 +110,7 @@ export async function build(
         effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`);
         outputs.set(path, {type: "module", resolvers});
         ++assetCount;
+        addToManifest("modules", path, module);
         continue;
       }
     }
@@ -99,6 +119,7 @@ export async function build(
       effects.output.write(`${faint("copy")} ${join(root, path)} ${faint("→")} `);
       const sourcePath = join(root, await file.load({useStale: true}, effects));
       await effects.copyFile(sourcePath, path);
+      addToManifest("files", path, file);
       ++assetCount;
       continue;
     }
@@ -209,7 +230,10 @@ export async function build(
   // Copy over referenced files, accumulating hashed aliases.
   for (const file of files) {
     effects.output.write(`${faint("copy")} ${join(root, file)} ${faint("→")} `);
-    const sourcePath = join(root, await loaders.loadFile(join("/", file), {useStale: true}, effects));
+    const path = join("/", file);
+    const loader = loaders.find(path);
+    if (!loader) throw enoent(path);
+    const sourcePath = join(root, await loader.load({useStale: true}, effects));
     const contents = await readFile(sourcePath);
     const hash = createHash("sha256").update(contents).digest("hex").slice(0, 8);
     const alias = applyHash(join("/_file", file), hash);
@@ -338,15 +362,13 @@ export async function build(
   }
 
   // Render pages!
-  const buildManifest: BuildManifest = {pages: []};
-  if (config.title) buildManifest.title = config.title;
   for (const [path, output] of outputs) {
     effects.output.write(`${faint("render")} ${path} ${faint("→")} `);
     if (output.type === "page") {
       const {page, resolvers} = output;
       const html = await renderPage(page, {...config, path, resolvers});
       await effects.writeFile(`${path}.html`, html);
-      buildManifest.pages.push({path: config.normalizePath(path), title: page.title});
+      addToManifest("pages", path, page);
     } else {
       const {resolvers} = output;
       const source = await renderModule(root, path, resolvers);
@@ -507,5 +529,8 @@ export class FileBuildEffects implements BuildEffects {
 
 export interface BuildManifest {
   title?: string;
-  pages: {path: string; title: string | null}[];
+  config: {root: string};
+  pages: {path: string; title?: string | null; source?: string}[];
+  modules: {path: string; source?: string}[];
+  files: {path: string; source?: string}[];
 }

src/client/stdlib/sqlite.js

@@ -1,10 +1,6 @@
-// https://github.com/sql-js/sql.js/issues/284
-const SQLite = await (async () => {
-  const exports = {};
-  const response = await fetch(import.meta.resolve("npm:sql.js/dist/sql-wasm.js"));
-  new Function("exports", await response.text())(exports);
-  return exports.Module({locateFile: (name) => import.meta.resolve("npm:sql.js/dist/") + name});
-})();
+import initSqlJs from "npm:sql.js";
+
+const SQLite = initSqlJs({locateFile: (name) => import.meta.resolve("npm:sql.js/dist/") + name});
 
 export default SQLite;
 
@@ -15,7 +11,8 @@ export class SQLiteDatabaseClient {
     });
   }
   static async open(source) {
-    return new SQLiteDatabaseClient(new SQLite.Database(await load(await source)));
+    const [sqlite, data] = await Promise.all([SQLite, Promise.resolve(source).then(load)]);
+    return new SQLiteDatabaseClient(new sqlite.Database(data));
   }
   async query(query, params) {
     return await exec(this._db, query, params);

src/files.ts

@@ -7,13 +7,8 @@ import {cwd} from "node:process";
 import {fileURLToPath} from "node:url";
 import {isEnoent} from "./error.js";
 
-export function toOsPath(path: string): string {
-  return path.split(sep).join(op.sep);
-}
-
-export function fromOsPath(path: string): string {
-  return path.split(op.sep).join(sep);
-}
+export const toOsPath = sep === op.sep ? (path: string) => path : (path: string) => path.split(sep).join(op.sep);
+export const fromOsPath = sep === op.sep ? (path: string) => path : (path: string) => path.split(op.sep).join(sep);
 
 /**
  * Returns the relative path from the current working directory to the given

src/loader.ts

@@ -73,25 +73,15 @@ export class LoaderResolver {
     );
   }
 
-  /**
-   * Loads the file at the specified path, returning a promise to the path to
-   * the (possibly generated) file relative to the source root.
-   */
-  async loadFile(path: string, options?: LoadOptions, effects?: LoadEffects): Promise<string> {
-    const loader = this.find(path);
-    if (!loader) throw enoent(path);
-    return await loader.load(options, effects);
-  }
-
   /**
    * Loads the page at the specified path, returning a promise to the parsed
    * page object.
    */
   async loadPage(path: string, options: LoadOptions & ParseOptions, effects?: LoadEffects): Promise<MarkdownPage> {
     const loader = this.findPage(path);
     if (!loader) throw enoent(path);
-    const source = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
-    return parseMarkdown(source, {params: loader.params, ...options});
+    const input = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
+    return parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
   }
 
   /**

src/markdown.ts

@@ -40,6 +40,7 @@ export interface MarkdownPage {
   data: FrontMatter;
   style: string | null;
   code: MarkdownCode[];
+  path: string;
   params?: Params;
 }
 
@@ -216,6 +217,7 @@ export interface ParseOptions {
   head?: Config["head"];
   header?: Config["header"];
   footer?: Config["footer"];
+  source?: string;
   params?: Params;
 }
 
@@ -242,7 +244,7 @@ export function createMarkdownIt({
 }
 
 export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
-  const {md, path, params} = options;
+  const {md, path, source = path, params} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
   const context: ParseContext = {code, startLine: 0, currentLine: 0, path, params};
@@ -258,6 +260,7 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
     title,
     style: getStyle(data, options),
     code,
+    path: source,
     params
   };
 }

src/npm.ts

@@ -83,14 +83,22 @@ export async function populateNpmCache(root: string, path: string): Promise<stri
   let promise = npmRequests.get(outputPath);
   if (promise) return promise; // coalesce concurrent requests
   promise = (async () => {
-    const specifier = extractNpmSpecifier(path);
+    let specifier = extractNpmSpecifier(path);
+    const s = parseNpmSpecifier(specifier);
+    // https://github.com/sql-js/sql.js/issues/284
+    if (s.name === "sql.js" && s.path === "+esm") {
+      specifier = formatNpmSpecifier({...s, path: "dist/sql-wasm.js"});
+    }
     const href = `https://cdn.jsdelivr.net/npm/${specifier}`;
     console.log(`npm:${specifier} ${faint("→")} ${outputPath}`);
     const response = await fetch(href);
     if (!response.ok) throw new Error(`unable to fetch: ${href}`);
     await mkdir(dirname(outputPath), {recursive: true});
     if (/^application\/javascript(;|$)/i.test(response.headers.get("content-type")!)) {
-      const source = await response.text();
+      let source = await response.text();
+      if (s.name === "sql.js" && s.path === "+esm") {
+        source = "var module;\n" + source + "\nexport default initSqlJs;";
+      }
       const resolver = await getDependencyResolver(root, path, source);
       await writeFile(outputPath, rewriteNpmImports(source, resolver), "utf-8");
     } else {

src/observableApiClient.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import type {BuildManifest} from "./build.js";
 import type {ClackEffects} from "./clack.js";
 import {CliError, HttpError, isApiError} from "./error.js";
 import {formatByteSize} from "./format.js";
@@ -229,7 +230,7 @@ export class ObservableApiClient {
     });
   }
 
-  async postDeployUploaded(deployId: string, buildManifest: PostDeployUploadedRequest | null): Promise<DeployInfo> {
+  async postDeployUploaded(deployId: string, buildManifest: BuildManifest | null): Promise<DeployInfo> {
     return await this._fetch<DeployInfo>(new URL(`/cli/deploy/${deployId}/uploaded`, this._apiOrigin), {
       method: "POST",
       headers: {"content-type": "application/json"},
@@ -383,10 +384,3 @@ export interface PostDeployManifestResponse {
     detail: string | null;
   }[];
 }
-
-export interface PostDeployUploadedRequest {
-  pages: {
-    path: string;
-    title: string | null;
-  }[];
-}

src/path.ts

@@ -1,4 +1,5 @@
-import {dirname, isAbsolute, join, normalize, relative, resolve} from "node:path/posix";
+import op from "node:path";
+import {dirname, join} from "node:path/posix";
 
 /**
  * Returns the normalized relative path from "/file/path/to/a" to
@@ -86,6 +87,7 @@ export function parseRelativeUrl(url: string): {pathname: string; search: string
 }
 
 export function within(root: string, path: string): boolean {
+  const {relative, normalize, resolve, isAbsolute} = op;
   path = relative(normalize(resolve(root)), normalize(resolve(path)));
   return !path.startsWith("..") && !isAbsolute(path);
 }

src/preview.ts

@@ -177,7 +177,10 @@ export class PreviewServer {
         }
         throw enoent(path);
       } else if (pathname.startsWith("/_file/")) {
-        send(req, await loaders.loadFile(pathname.slice("/_file".length)), {root}).pipe(res);
+        const path = pathname.slice("/_file".length);
+        const loader = loaders.find(path);
+        if (!loader) throw enoent(path);
+        send(req, await loader.load(), {root}).pipe(res);
       } else {
         if ((pathname = normalize(pathname)).startsWith("..")) throw new Error("Invalid path: " + pathname);
 

src/resolvers.ts

@@ -14,7 +14,8 @@ import type {LoaderResolver} from "./loader.js";
 import type {MarkdownPage} from "./markdown.js";
 import {extractNodeSpecifier, resolveNodeImport, resolveNodeImports} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport, resolveNpmImports} from "./npm.js";
-import {isAssetPath, isPathImport, parseRelativeUrl, relativePath, resolveLocalPath, resolvePath} from "./path.js";
+import {isAssetPath, isPathImport, parseRelativeUrl} from "./path.js";
+import {relativePath, resolveLocalPath, resolvePath, resolveRelativePath} from "./path.js";
 
 export interface Resolvers {
   path: string;
@@ -133,7 +134,7 @@ export async function getResolvers(page: MarkdownPage, config: ResolversConfig):
     for (const value of Object.values(page.data.sql)) {
       const source = String(value);
       if (isAssetPath(source)) {
-        files.add(source);
+        files.add(resolveRelativePath(path, source));
       }
     }
   }

src/style/layout.css

@@ -123,7 +123,7 @@ body {
   visibility: hidden;
   font-weight: 500;
   width: calc(272px + var(--observablehq-sidebar-padding-left));
-  z-index: 2;
+  z-index: 3;
   top: 0;
   bottom: 0;
   left: -272px;