Merge pull request #184 from flatironinstitute/reduce-redundant-compi…

…le-jobs

reduce redundant compile jobs, cache main.js URL

gui/src/app/FileEditor/StanFileEditor.tsx

@@ -79,16 +79,6 @@ const StanFileEditor: FunctionComponent<Props> = ({
     setCompiledUrl(mainJsUrl);
     setCompileStatus("compiled");
     setTheStanFileContentThasHasBeenCompiled(fileContent);
-
-    // record in local storage that we compiled this particular stan file
-    try {
-      const key = getKeyNameForCompiledFile(stanWasmServerUrl, fileContent);
-      const value = JSON.stringify({ mainJsUrl });
-      localStorage.setItem(key, value);
-    } catch (e: any) {
-      console.error("Problem recording compiled file in local storage");
-      console.error(e);
-    }
   }, [fileContent, setCompiledUrl]);
 
   useEffect(() => {
@@ -107,22 +97,6 @@ const StanFileEditor: FunctionComponent<Props> = ({
     setCompiledUrl,
   ]);
 
-  const [didInitialCompile, setDidInitialCompile] = useState(false);
-  useEffect(() => {
-    // if we think this has been compiled before, let's go ahead and compile it (should be in cache on server)
-    // but we are only going to do this on initial load
-    if (didInitialCompile) return;
-    const stanWasmServerUrl = localStorage.getItem("stanWasmServerUrl") || "";
-    if (!stanWasmServerUrl) return;
-    const key = getKeyNameForCompiledFile(stanWasmServerUrl, fileContent);
-    const value = localStorage.getItem(key);
-    if (!value) return;
-    handleCompile();
-    if (fileContent) {
-      setDidInitialCompile(true);
-    }
-  }, [fileContent, handleCompile, didInitialCompile]);
-
   const [syntaxWindowVisible, setSyntaxWindowVisible] = useState(false);
 
   const toolbarItems: ToolbarItem[] = useMemo(() => {
@@ -243,22 +217,4 @@ const StanFileEditor: FunctionComponent<Props> = ({
   );
 };
 
-const getKeyNameForCompiledFile = (
-  stanWasmServerUrl: string,
-  stanFileContent: string,
-) => {
-  return `compiled-file|${stanWasmServerUrl}|${stringChecksum(stanFileContent)}`;
-};
-
-const stringChecksum = (str: string) => {
-  let hash = 0;
-  if (str.length == 0) return hash;
-  for (let i = 0; i < str.length; i++) {
-    const char = str.charCodeAt(i);
-    hash = (hash << 5) - hash + char;
-    hash = hash & hash; // Convert to 32bit integer
-  }
-  return hash;
-};
-
 export default StanFileEditor;

gui/src/app/compileStanProgram/compileStanProgram.ts

@@ -1,9 +1,21 @@
+import { checkMainJsUrlCache, setToMainJsUrlCache } from "./mainJsUrlCache";
+
 const compileStanProgram = async (
   stanWasmServerUrl: string,
   stanProgram: string,
   onStatus: (s: string) => void,
 ): Promise<{ mainJsUrl?: string }> => {
   try {
+    onStatus("checking cache");
+    const downloadMainJsUrlFromCache = await checkMainJsUrlCache(
+      stanProgram,
+      stanWasmServerUrl,
+    );
+    if (downloadMainJsUrlFromCache) {
+      onStatus("compiled");
+      return { mainJsUrl: downloadMainJsUrlFromCache };
+    }
+
     onStatus("initiating job");
     const initiateJobUrl = `${stanWasmServerUrl}/job/initiate`;
     // post
@@ -56,6 +68,8 @@ const compileStanProgram = async (
 
     const downloadMainJsUrl = `${stanWasmServerUrl}/job/${job_id}/download/main.js`;
 
+    setToMainJsUrlCache(stanProgram, downloadMainJsUrl);
+
     // download to make sure it is there
     onStatus("Checking download of main.js");
     const d = await fetch(downloadMainJsUrl);

gui/src/app/compileStanProgram/mainJsUrlCache.ts

@@ -0,0 +1,73 @@
+export const checkMainJsUrlCache = async (
+  stanProgram: string,
+  baseStanWasmServerUrl: string,
+): Promise<string | null> => {
+  const mainJsCache = localStorage.getItem("mainJsCache");
+  if (!mainJsCache) return null;
+  try {
+    const cache = JSON.parse(mainJsCache);
+    const stanProgramHash = stringChecksum(stanProgram);
+    const url = cache[stanProgramHash];
+    if (!url) return null;
+    if (!url.startsWith(baseStanWasmServerUrl)) {
+      // the url is not from the current server
+      return null;
+    }
+    // check to see if the url is still valid
+    const exists = await checkRemoteFileExists(url);
+    if (!exists) {
+      delete cache[stanProgramHash];
+      localStorage.setItem("mainJsCache", JSON.stringify(cache));
+      return null;
+    }
+    return url;
+  } catch (e) {
+    console.error("Problem parsing mainJsCache");
+    console.error(e);
+    try {
+      localStorage.removeItem("mainJsCache");
+    } catch (e) {
+      console.error("Problem removing mainJsCache");
+      console.error(e);
+    }
+    return null;
+  }
+};
+
+export const setToMainJsUrlCache = (stanProgram: string, url: string) => {
+  try {
+    const mainJsCache = localStorage.getItem("mainJsCache");
+    const stanProgramHash = stringChecksum(stanProgram);
+    let cache = mainJsCache ? JSON.parse(mainJsCache) : {};
+    cache = cleanupIfGettingTooBig(cache);
+    cache[stanProgramHash] = url;
+    localStorage.setItem("mainJsCache", JSON.stringify(cache));
+  } catch (e) {
+    console.error("Problem setting mainJsCache");
+    console.error(e);
+  }
+};
+
+const cleanupIfGettingTooBig = (cache: { [key: string]: string }) => {
+  // for now we just clear the whole thing if it's getting too big
+  if (Object.keys(cache).length > 50) {
+    return {};
+  }
+  return cache;
+};
+
+const stringChecksum = (str: string) => {
+  let hash = 0;
+  if (str.length == 0) return hash;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash; // Convert to 32bit integer
+  }
+  return hash;
+};
+
+const checkRemoteFileExists = async (url: string): Promise<boolean> => {
+  const response = await fetch(url, { method: "HEAD" });
+  return response.ok;
+};