Configure storage used for execution. Closes #1263 (#1285)

deepforge-dev · Oct 18, 2019 · ec2b097 · ec2b097
1 parent 97aa40b
commit ec2b097
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 3 deletions.
diff --git a/src/common/viz/Execute.js b/src/common/viz/Execute.js
@@ -3,6 +3,7 @@
 define([
     'q',
     'deepforge/compute/index',
+    'deepforge/storage/index',
     'deepforge/viz/ConfigDialog',
     'deepforge/api/ExecPulseClient',
     'deepforge/api/JobOriginClient',
@@ -11,6 +12,7 @@ define([
 ], function(
     Q,
     Compute,
+    Storage,
     ConfigDialog,
     ExecPulseClient,
     JobOriginClient,
@@ -71,6 +73,21 @@ define([
                 .map(id => Compute.getMetadata(id)),
         });
 
+        metadata.configStructure.push({
+            name: 'storageHeader',
+            displayName: 'Storage Options',
+            valueType: 'section'
+        });
+        metadata.configStructure.push({
+            name: 'storage',
+            displayName: 'Storage',
+            description: 'Location to store intermediate/generated data.',
+            valueType: 'dict',
+            value: Storage.getBackend(Storage.getAvailableBackends()[0]).name,
+            valueItems: Storage.getAvailableBackends()
+                .map(id => Storage.getStorageMetadata(id)),
+        });
+
         const allConfigs = await configDialog.show(metadata);
         context.pluginConfig = allConfigs[pluginId];
 

diff --git a/src/plugins/GenerateJob/GenerateJob.js b/src/plugins/GenerateJob/GenerateJob.js
@@ -293,10 +293,16 @@ define([
         const inputs = allInputs
             .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
 
+        const storage = this.getCurrentConfig().storage || {};
+        storage.id = storage.id || 'gme';
+        storage.config = storage.config || {};
+
         const startJS = _.template(Templates.START)({
             CONSTANTS,
-            inputs: inputs.map(pair => pair[0])
+            storageId: storage.id,
+            inputs: inputs.map(pair => pair[0]),
         });
+        files.addFile('storage-config.json', JSON.stringify(storage.config));
         files.addFile('start.js', startJS);
         files.addFile('backend_deepforge.py', Templates.MATPLOTLIB_BACKEND);
 

diff --git a/src/plugins/GenerateJob/templates/start.ejs b/src/plugins/GenerateJob/templates/start.ejs
@@ -266,8 +266,10 @@ requirejs([
                 exitCode = code;
                 log('script finished w/ exit code:', code);
                 if (exitCode === 0) {
-                    const results = JSON.parse(fs.readFileSync('result-types.json'));
-                    const client = await Storage.getBackend('gme').getClient(logger);
+                    const results = require('./result-types.json');
+                    const STORAGE_ID = '<%= storageId %>';
+                    const config = require('./storage-config.json');
+                    const client = await Storage.getBackend(STORAGE_ID).getClient(logger, config);
                     const outputNames = Object.keys(results);
 
                     for (let i = outputNames.length; i--;) {