find fetches in locally imported files (#286)

* wip

* working for both build and dev

* delint

* fix tests

* fix typescript errors

* redelint

* fix type import

* add local fetch test

* some more tests

* better describe test

* fix wording

* properly handle fetches directly from markdown

* delint and fix typescript

* fix tests

* fix missing fetch test output

* rename rewriteFetch -> rewriteIfLocalFetch

* improve typescript typing

* extract referenceds in imports.ts and adjust fetch logic to accept references directly

* move reference search into findFetches to better findImports pattern

* remove blank line

* update output

* change maybeExtractFetch -> maybeAddFetch and pass in target array

src/javascript.ts

@@ -138,13 +138,14 @@ function parseJavaScript(input: string, options: ParseOptions): JavaScriptNode {
   const references = findReferences(body, globals);
   findAssignments(body, references, globals, input);
   const declarations = expression ? null : findDeclarations(body as Program, globals, input);
-  const imports = findImports(body, root, sourcePath);
+  const {imports, fetches} = findImports(body, root, sourcePath);
   const features = findFeatures(body, root, sourcePath, references, input);
+
   return {
     body,
     declarations,
     references,
-    features,
+    features: [...features, ...fetches],
     imports,
     expression: !!expression,
     async: findAwaits(body).length > 0

src/javascript/features.ts

@@ -1,8 +1,7 @@
-import type {CallExpression, Identifier, Literal, Node, TemplateLiteral} from "acorn";
+import type {Identifier, Literal, Node, TemplateLiteral} from "acorn";
 import {simple} from "acorn-walk";
 import {getLocalPath} from "../files.js";
 import type {Feature} from "../javascript.js";
-import {isLocalImport} from "./imports.js";
 import {syntaxError} from "./syntaxError.js";
 
 export function findFeatures(
@@ -22,12 +21,6 @@ export function findFeatures(
         arguments: [arg]
       } = node;
 
-      // Promote fetches with static literals to file attachment references.
-      if (isLocalFetch(node, references, sourcePath)) {
-        features.push({type: "FileAttachment", name: getStringLiteralValue(arg)});
-        return;
-      }
-
       // Ignore function calls that are not references to the feature. For
       // example, if there’s a local variable called Secret, that will mask the
       // built-in Secret and won’t be considered a feature.
@@ -57,20 +50,6 @@ export function findFeatures(
   return features;
 }
 
-export function isLocalFetch(node: CallExpression, references: Identifier[], sourcePath: string): boolean {
-  const {
-    callee,
-    arguments: [arg]
-  } = node;
-  return (
-    callee.type === "Identifier" &&
-    callee.name === "fetch" &&
-    !references.includes(callee) &&
-    isStringLiteral(arg) &&
-    isLocalImport(getStringLiteralValue(arg), sourcePath)
-  );
-}
-
 export function isStringLiteral(node: any): node is Literal | TemplateLiteral {
   return (
     node &&

src/javascript/fetches.ts

@@ -1,18 +1,71 @@
+import type {CallExpression, Identifier, Node} from "acorn";
 import {simple} from "acorn-walk";
-import {type JavaScriptNode} from "../javascript.js";
+import {type Feature, type JavaScriptNode} from "../javascript.js";
 import {type Sourcemap} from "../sourcemap.js";
 import {relativeUrl, resolvePath} from "../url.js";
-import {getStringLiteralValue, isLocalFetch} from "./features.js";
+import {getStringLiteralValue, isStringLiteral} from "./features.js";
+import {defaultGlobals} from "./globals.js";
+import {isLocalImport} from "./imports.js";
+import {findReferences} from "./references.js";
 
 export function rewriteFetches(output: Sourcemap, rootNode: JavaScriptNode, sourcePath: string): void {
   simple(rootNode.body, {
     CallExpression(node) {
-      if (isLocalFetch(node, rootNode.references, sourcePath)) {
-        const arg = node.arguments[0];
-        const value = getStringLiteralValue(arg);
-        const path = resolvePath("_file", sourcePath, value);
-        output.replaceLeft(arg.start, arg.end, JSON.stringify(relativeUrl(sourcePath, path)));
-      }
+      rewriteIfLocalFetch(node, output, rootNode.references, sourcePath);
     }
   });
 }
+
+export function rewriteIfLocalFetch(
+  node: CallExpression,
+  output: Sourcemap,
+  references: Identifier[],
+  sourcePath: string
+) {
+  if (isLocalFetch(node, references, sourcePath)) {
+    const arg = node.arguments[0];
+    const value = getStringLiteralValue(arg);
+    const path = resolvePath("_file", sourcePath, value);
+    output.replaceLeft(arg.start, arg.end, JSON.stringify(relativeUrl(sourcePath, path)));
+  }
+}
+
+export function findFetches(body: Node, path: string) {
+  const references: Identifier[] = findReferences(body, defaultGlobals);
+  const fetches: Feature[] = [];
+
+  simple(body, {CallExpression: findFetch}, undefined, path);
+
+  // Promote fetches with static literals to file attachment references.
+
+  function findFetch(node: CallExpression, sourcePath: string) {
+    maybeAddFetch(fetches, node, references, sourcePath);
+  }
+
+  return fetches;
+}
+
+export function maybeAddFetch(
+  features: Feature[],
+  node: CallExpression,
+  references: Identifier[],
+  sourcePath: string
+): void {
+  if (isLocalFetch(node, references, sourcePath)) {
+    features.push({type: "FileAttachment", name: getStringLiteralValue(node.arguments[0])});
+  }
+}
+
+function isLocalFetch(node: CallExpression, references: Identifier[], sourcePath: string): boolean {
+  const {
+    callee,
+    arguments: [arg]
+  } = node;
+  return (
+    callee.type === "Identifier" &&
+    callee.name === "fetch" &&
+    !references.includes(callee) &&
+    isStringLiteral(arg) &&
+    isLocalImport(getStringLiteralValue(arg), sourcePath)
+  );
+}

src/javascript/imports.ts

@@ -2,13 +2,31 @@ import {createHash} from "node:crypto";
 import {readFileSync} from "node:fs";
 import {join} from "node:path";
 import {Parser} from "acorn";
-import type {ExportAllDeclaration, ExportNamedDeclaration, ImportDeclaration, ImportExpression, Node} from "acorn";
+import type {
+  CallExpression,
+  ExportAllDeclaration,
+  ExportNamedDeclaration,
+  Identifier,
+  ImportDeclaration,
+  ImportExpression,
+  Node,
+  Program
+} from "acorn";
 import {simple} from "acorn-walk";
 import {isEnoent} from "../error.js";
-import {type ImportReference, type JavaScriptNode, parseOptions} from "../javascript.js";
+import {type Feature, type ImportReference, type JavaScriptNode} from "../javascript.js";
+import {parseOptions} from "../javascript.js";
 import {Sourcemap} from "../sourcemap.js";
 import {relativeUrl, resolvePath} from "../url.js";
 import {getStringLiteralValue, isStringLiteral} from "./features.js";
+import {findFetches, maybeAddFetch, rewriteIfLocalFetch} from "./fetches.js";
+import {defaultGlobals} from "./globals.js";
+import {findReferences} from "./references.js";
+
+export interface ImportsAndFetches {
+  imports: ImportReference[];
+  fetches: Feature[];
+}
 
 /**
  * Finds all export declarations in the specified node. (This is used to
@@ -34,13 +52,17 @@ export function findExports(body: Node): (ExportAllDeclaration | ExportNamedDecl
  * Recursively processes any imported local ES modules. The returned transitive
  * import paths are relative to the given source path.
  */
-export function findImports(body: Node, root: string, path: string): ImportReference[] {
+
+export function findImports(body: Node, root: string, path: string): ImportsAndFetches {
+  const references: Identifier[] = findReferences(body, defaultGlobals);
   const imports: ImportReference[] = [];
+  const fetches: Feature[] = [];
   const paths: string[] = [];
 
   simple(body, {
     ImportDeclaration: findImport,
-    ImportExpression: findImport
+    ImportExpression: findImport,
+    CallExpression: findFetch
   });
 
   function findImport(node) {
@@ -54,8 +76,14 @@ export function findImports(body: Node, root: string, path: string): ImportRefer
     }
   }
 
+  function findFetch(node) {
+    maybeAddFetch(fetches, node, references, path);
+  }
+
   // Recursively process any imported local ES modules.
-  imports.push(...parseLocalImports(root, paths));
+  const features = parseLocalImports(root, paths);
+  imports.push(...features.imports);
+  fetches.push(...features.fetches);
 
   // Make all local paths relative to the source path.
   for (const i of imports) {
@@ -64,7 +92,7 @@ export function findImports(body: Node, root: string, path: string): ImportRefer
     }
   }
 
-  return imports;
+  return {imports, fetches};
 }
 
 /**
@@ -73,14 +101,16 @@ export function findImports(body: Node, root: string, path: string): ImportRefer
  * appends to imports. The paths here are always relative to the root (unlike
  * findImports above!).
  */
-export function parseLocalImports(root: string, paths: string[]): ImportReference[] {
+export function parseLocalImports(root: string, paths: string[]): ImportsAndFetches {
   const imports: ImportReference[] = [];
+  const fetches: Feature[] = [];
   const set = new Set(paths);
   for (const path of set) {
     imports.push({type: "local", name: path});
     try {
       const input = readFileSync(join(root, path), "utf-8");
-      const program = Parser.parse(input, parseOptions);
+      const program = Parser.parse(input, parseOptions) as Program;
+
       simple(
         program,
         {
@@ -92,6 +122,7 @@ export function parseLocalImports(root: string, paths: string[]): ImportReferenc
         undefined,
         path
       );
+      fetches.push(...findFetches(program, path));
     } catch (error) {
       if (!isEnoent(error) && !(error instanceof SyntaxError)) throw error;
     }
@@ -110,19 +141,23 @@ export function parseLocalImports(root: string, paths: string[]): ImportReferenc
       }
     }
   }
-  return imports;
+  return {imports, fetches};
 }
 
 /** Rewrites import specifiers in the specified ES module source. */
 export function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): string {
-  const body = Parser.parse(input, parseOptions);
+  const body = Parser.parse(input, parseOptions) as Program;
+  const references: Identifier[] = findReferences(body, defaultGlobals);
   const output = new Sourcemap(input);
 
   simple(body, {
     ImportDeclaration: rewriteImport,
     ImportExpression: rewriteImport,
     ExportAllDeclaration: rewriteImport,
-    ExportNamedDeclaration: rewriteImport
+    ExportNamedDeclaration: rewriteImport,
+    CallExpression(node: CallExpression) {
+      rewriteIfLocalFetch(node, output, references, sourcePath);
+    }
   });
 
   function rewriteImport(node: ImportDeclaration | ImportExpression | ExportAllDeclaration | ExportNamedDeclaration) {
@@ -210,7 +245,8 @@ function getModuleHash(root: string, path: string): string {
     if (!isEnoent(error)) throw error;
   }
   // TODO can’t simply concatenate here; we need a delimiter
-  for (const i of parseLocalImports(root, [path])) {
+  const {imports, fetches} = parseLocalImports(root, [path]);
+  for (const i of [...imports, ...fetches]) {
     if (i.type === "local") {
       try {
         hash.update(readFileSync(join(root, i.name), "utf-8"));

test/input/build/fetches/foo.md

@@ -0,0 +1,8 @@
+# Top
+
+```js
+import {fooJsonData, fooCsvData} from "/foo/foo.js";
+
+display(fooJsonData);
+display(fooCsvData);
+```

test/input/build/fetches/foo/foo-data.csv

@@ -0,0 +1,3 @@
+eruid,description
+batman,uses technology
+superman,flies through the air

test/input/build/fetches/foo/foo-data.json

@@ -0,0 +1 @@
+[1, 2, 3]

test/input/build/fetches/foo/foo.js

@@ -0,0 +1,2 @@
+export const fooJsonData = await fetch("./foo-data.json").then(d => d.json());
+export const fooCsvData = await fetch("./foo-data.csv").then(d => d.csv({ typed: true }));

test/input/build/fetches/top-data.csv

@@ -0,0 +1,3 @@
+eruid,description
+batman,uses technology
+superman,flies through the air

test/input/build/fetches/top-data.json

@@ -0,0 +1 @@
+[1, 2, 3]

test/input/build/fetches/top.js

@@ -0,0 +1,3 @@
+export {fooCsvData, fooJsonData} from "./foo/foo.js";
+export const topJsonData = await fetch("./top-data.json").then(d => d.json());
+export const topCsvData = await fetch("./top-data.csv").then(d => d.csv({ typed: true }));

test/input/build/fetches/top.md

@@ -0,0 +1,10 @@
+# Top
+
+```js
+import {fooCsvData, fooJsonData, topCsvData, topjsondata} from "/top.js";
+
+display(fooJsonData);
+display(fooCsvData);
+display(topJsonData);
+display(topCsvData);
+```

test/input/imports/baz.js

@@ -0,0 +1 @@
+export const data = fetch("./fetch-local-data.json").then(d => d.json());

test/input/imports/local-fetch-data.json

@@ -0,0 +1 @@
+[1, 2, 3]

test/input/imports/local-fetch-from-import.js

@@ -0,0 +1,3 @@
+import {data} from "./baz.js";
+
+display(data);

test/javascript/fetches-test.ts

@@ -0,0 +1,25 @@
+import assert from "node:assert";
+import {ascending} from "d3-array";
+import {parseLocalImports} from "../../src/javascript/imports.js";
+import {type Feature} from "../../src/javascript.js";
+
+describe("parseLocalFetches(root, paths)", () => {
+  it("find all local fetches in one file", () => {
+    assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["foo/foo.js"]).fetches.sort(compareImport), [
+      {name: "./foo-data.csv", type: "FileAttachment"},
+      {name: "./foo-data.json", type: "FileAttachment"}
+    ]);
+  });
+  it("find all local fetches via transivite import", () => {
+    assert.deepStrictEqual(parseLocalImports("test/input/build/fetches", ["top.js"]).fetches.sort(compareImport), [
+      {name: "./foo-data.csv", type: "FileAttachment"},
+      {name: "./foo-data.json", type: "FileAttachment"},
+      {name: "./top-data.csv", type: "FileAttachment"},
+      {name: "./top-data.json", type: "FileAttachment"}
+    ]);
+  });
+});
+
+function compareImport(a: Feature, b: Feature): number {
+  return ascending(a.type, b.type) || ascending(a.name, b.name);
+}

test/javascript/imports-test.ts

@@ -5,7 +5,7 @@ import {type ImportReference} from "../../src/javascript.js";
 
 describe("parseLocalImports(root, paths)", () => {
   it("finds all local imports in one file", () => {
-    assert.deepStrictEqual(parseLocalImports("test/input/build/imports", ["foo/foo.js"]).sort(compareImport), [
+    assert.deepStrictEqual(parseLocalImports("test/input/build/imports", ["foo/foo.js"]).imports.sort(compareImport), [
       {name: "npm:d3", type: "global"},
       {name: "bar/bar.js", type: "local"},
       {name: "bar/baz.js", type: "local"},
@@ -15,7 +15,9 @@ describe("parseLocalImports(root, paths)", () => {
   });
   it("finds all local imports in multiple files", () => {
     assert.deepStrictEqual(
-      parseLocalImports("test/input/imports", ["transitive-static-import.js", "dynamic-import.js"]).sort(compareImport),
+      parseLocalImports("test/input/imports", ["transitive-static-import.js", "dynamic-import.js"]).imports.sort(
+        compareImport
+      ),
       [
         {name: "bar.js", type: "local"},
         {name: "dynamic-import.js", type: "local"},
@@ -26,7 +28,7 @@ describe("parseLocalImports(root, paths)", () => {
   });
   it("ignores missing files", () => {
     assert.deepStrictEqual(
-      parseLocalImports("test/input/imports", ["static-import.js", "does-not-exist.js"]).sort(compareImport),
+      parseLocalImports("test/input/imports", ["static-import.js", "does-not-exist.js"]).imports.sort(compareImport),
       [
         {name: "bar.js", type: "local"},
         {name: "does-not-exist.js", type: "local"},

test/output/build/fetches/_file/top-data.csv

@@ -0,0 +1,3 @@
+eruid,description
+batman,uses technology
+superman,flies through the air

test/output/build/fetches/_file/top-data.json

@@ -0,0 +1 @@
+[1, 2, 3]

test/output/build/fetches/_import/foo/foo.js

@@ -0,0 +1,2 @@
+export const fooJsonData = await fetch("../_file/foo/foo-data.json").then(d => d.json());
+export const fooCsvData = await fetch("../_file/foo/foo-data.csv").then(d => d.csv({ typed: true }));

test/output/build/fetches/_import/top.js

@@ -0,0 +1,3 @@
+export {fooCsvData, fooJsonData} from "./foo/foo.js?sha=7a93b271ec78dd07db6d9265e7b82eacc1a1bb6682cd665f0f86ea2c0fbc7350";
+export const topJsonData = await fetch("./_file/top-data.json").then(d => d.json());
+export const topCsvData = await fetch("./_file/top-data.csv").then(d => d.csv({ typed: true }));