File based config persistence

.dockerignore

@@ -0,0 +1,13 @@
+# Items that don't need to be in a Docker image.
+# Anything not used by the build system should go here.
+Dockerfile
+.dockerignore
+.gitignore
+README.md
+
+# Artifacts that will be built during image creation.
+# This should contain all files created during `npm run build`.
+build
+node_modules
+server/config/*
+server/node_modules

.gitignore

@@ -26,4 +26,6 @@ yarn-error.log*
 #amplify
 build/
 dist/
-node_modules/
+node_modules/
+
+server/config/*

server/index.js

@@ -1,31 +1,103 @@
 const express = require('express');
 const path = require('path');
 const fs = require('fs');
-// const auth = require('koa-basic-auth');
-// const Router = require('koa-router');
+const fsExtra = require('fs-extra');
+const auth = require('basic-auth');
 const app = express();
+require('dotenv').config()
 
-// const basicAuth = require('express-basic-auth')
-// const adminUser = process.env.ADMIN_USER || 'admin';
-// const adminPassword = process.env.ADMIN_PASSWORD || 'password';
+const CONFIG_FOLDER = process.env.CONFIG_FOLDER || '.';
 
 app.use(express.static(path.join(__dirname, '../build')));
 
-app.get('/*', function (req, res) {
-  res.sendFile(path.join(__dirname, '../build', 'index.html'));
-});
-
 
 // app.use(basicAuth({
 //     users: { [adminUser]: adminPassword }
 // }))
+
 app.use(express.json())
 
 
 app.post('/saveToFile', function (req, res) {
-  const configSavePath = process.env.REACT_APP_SAVE_TO_FILE_PATH || './config.json';
+  const configSavePath = CONFIG_FOLDER + '/workspaceConfig.json';
   fs.writeFileSync(configSavePath, JSON.stringify(req.body.workspaceConfig));
+  Object.entries(req.body.sourcesConfig).forEach(([writeKey, config])=>{
+    fsExtra.outputFileSync(`${CONFIG_FOLDER}/sources/${writeKey}.json`, JSON.stringify(config));
+  });
   res.send('Saved to file');
 });
 
+app.get('/workspaceConfig', function (req, res) {
+  const configPath = CONFIG_FOLDER + '/workspaceConfig.json';
+  if (fs.existsSync(configPath)) {
+    const workspaceConfig =  JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    res.send(workspaceConfig);
+  } else {
+    res.status(500);
+    res.send('Config not found');
+  }
+});
+
+
+app.get('/sourceConfig', function (req, res) {
+  const bearerHeader = req.headers.authorization;
+  const writeKey = auth.parse(bearerHeader) && auth.parse(bearerHeader).name;
+
+  const configPath = `${CONFIG_FOLDER}/sources/${writeKey}.json`;
+  if (fs.existsSync(configPath)) {
+    const sourceConfig =  JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    res.send(sourceConfig);
+  } else {
+    res.status(500);
+    res.send('Config not found');
+  }
+});
+
+const loadJSONFile = (path) => {
+  if (fs.existsSync(path)) {
+    return JSON.parse(fs.readFileSync(path, 'utf8'));
+  } else {
+    return "{}";
+  }
+}
+
+app.get('/loadConnections', function(req, res) {
+  const connections = loadJSONFile(CONFIG_FOLDER+'/connections.json');
+  res.send({connections});
+});
+
+app.get('/loadSources', function(req, res) {
+  const sources = loadJSONFile(CONFIG_FOLDER+'/sources.json');
+  res.send({sources});
+});
+
+app.get('/loadDestinations', function(req, res) {
+  const destinations = loadJSONFile(CONFIG_FOLDER+'/destinations.json');
+  res.send({destinations});
+});
+
+
+app.post('/saveConnections', function (req, res) {
+  const connectionsSavePath = CONFIG_FOLDER+'/connections.json';
+  fs.writeFileSync(connectionsSavePath, JSON.stringify(req.body.connections));
+  res.send('Saved connections to file');
+});
+
+app.post('/saveDestinations', function (req, res) {
+  const destinationsSavePath = CONFIG_FOLDER+'/destinations.json';
+  fs.writeFileSync(destinationsSavePath, JSON.stringify(req.body.destinations));
+  res.send('Saved destinations to file');
+});
+
+app.post('/saveSources', function (req, res) {
+  const sourcesSavePath = CONFIG_FOLDER+'/sources.json';
+  fs.writeFileSync(sourcesSavePath, JSON.stringify(req.body.sources));
+  res.send('Saved sources to file');
+});
+
+
+app.get('/*', function (req, res) {
+  res.sendFile(path.join(__dirname, '../build', 'index.html'));
+});
+
 app.listen(9000);

server/package.json

@@ -5,11 +5,15 @@
   "main": "index.js",
   "scripts": {
     "start": "node index.js",
+    "start-dev": "nodemon index.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "express": "^4.17.1"
+    "basic-auth": "^2.0.1",
+    "dotenv": "^10.0.0",
+    "express": "^4.17.1",
+    "fs-extra": "^10.0.0"
   }
 }

src/components/connections/index.tsx

@@ -5,6 +5,7 @@ import theme from '@css/theme';
 import { IDestinationsListStore } from '@stores/destinationsList';
 import { ISourceDefinitionsListStore } from '@stores/sourceDefinitionsList';
 import { ISourcesListStore } from '@stores/sourcesList';
+import { ISourceStore } from '@stores/source';
 import { IConnectionsStore } from '@stores/connections';
 import { inject, observer } from 'mobx-react';
 import React, { Component } from 'react';
@@ -93,7 +94,8 @@ class Connections extends Component<IConnectionsProps, any> {
       sources: [] as any,
       metadata: {
         sourceListStore: this.props.sourcesListStore.returnWithoutRootStore(),
-        destinationListStore: this.props.destinationsListStore.returnWithoutRootStore(),
+        destinationListStore:
+          this.props.destinationsListStore.returnWithoutRootStore(),
         connectionsStore: this.props.connectionsStore.returnWithoutRootStore(),
         version,
       },
@@ -137,8 +139,13 @@ class Connections extends Component<IConnectionsProps, any> {
   };
 
   handleSaveWorkspaceConfig = () => {
+    const { sourcesListStore } = this.props;
     const workspaceConfig = this.buildWorkspaceConfig();
-    apiServerCaller().post('/saveToFile', { workspaceConfig });
+    let sourcesConfig: any = {};
+    sourcesListStore.sources.forEach((source: ISourceStore) => {
+      sourcesConfig[source.writeKey] = source.configForSDK;
+    });
+    apiServerCaller().post('/saveToFile', { workspaceConfig, sourcesConfig });
   };
 
   handleFileChosen = (event: any) => {

src/components/home.tsx

@@ -112,9 +112,9 @@ class Home extends Component<IHomeProps> {
       sourceDefinitionsListStore,
       destinationDefsListStore,
     } = this.props;
-    sourcesListStore.loadAndSave();
-    destinationsListStore.loadAndSave();
-    connectionsStore.loadAndSave();
+    await sourcesListStore.loadAndSave();
+    await destinationsListStore.loadAndSave();
+    await connectionsStore.loadAndSave();
     await Promise.all([
       sourceDefinitionsListStore.getSourceDefinitions(),
       destinationDefsListStore.getDestinationDefs(),

src/components/icons/sourceIcon.tsx

@@ -7,7 +7,7 @@ import { ReactComponent as ClickHouse } from '@svg/clickhouse.svg';
 import { ReactComponent as Extole } from '@svg/extole.svg';
 import { ReactComponent as FacebookAds } from '@svg/facebook_ads.svg';
 import { ReactComponent as Flutter } from '@svg/flutter.svg';
-import { ReactComponent as FreshDesk } from '@svg/freshDesk.svg';
+import { ReactComponent as FreshDesk } from '@svg/freshdesk.svg';
 import { ReactComponent as GoogleAdwords } from '@svg/googleads.svg';
 import { ReactComponent as GoogleAnalytics } from '@svg/googleanalytics.svg';
 import { ReactComponent as GoogleSearchConsole } from '@svg/google_search_console.svg';

src/components/sourceDetails/index.tsx

@@ -106,40 +106,8 @@ class SourceDetails extends Component<IConfiguredSourcesProps, any> {
     const { sources } = sourcesListStore;
     const source = sources.find(source => source.id === sourceId);
     if (source) {
-      const sourceConfig = {
-        source: {
-          config: source.config,
-          id: source.id,
-          name: source.name,
-          writeKey: source.writeKey,
-          enabled: source.enabled,
-          sourceDefinitionId: source.sourceDefinitionId,
-          deleted: false,
-          createdAt: Date(),
-          updatedAt: Date(),
-          sourceDefinition: source.sourceDef,
-          // Filter only useNativeSDK enabled destinations and
-          // includes only includeKeys (from definition) in the config
-          destinations: source.destinations
-            .filter(dest => {
-              return dest.config ? dest.config.useNativeSDK : false;
-            })
-            .map(dest => {
-              return {
-                id: dest.id,
-                name: dest.name,
-                enabled: dest.enabled,
-                config: dest.filteredConfig(), // Very Very Important to use filterConfig instead of config
-                destinationDefinition: dest.destinationDefinition,
-              };
-            }),
-        },
-        metadata: {
-          version: version,
-        },
-      };
       fileDownload(
-        JSON.stringify(sourceConfig, null, 2),
+        JSON.stringify(source.configForSDK, null, 2),
         `${source.name}_Source_Config.json`,
       );
     }

src/services/config.ts

@@ -0,0 +1,7 @@
+export interface IConfig {
+  persistanceMode: string;
+}
+
+export const config: IConfig  = {
+  persistanceMode: process.env.REACT_APP_PERSISTANCE_MODE || 'localStorage'
+}

src/stores/connections.ts

@@ -3,6 +3,8 @@ import { action, observable, trace, autorun, set, toJS } from 'mobx';
 import { IRootStore } from '.';
 import { ISourceStore } from './source';
 import { IDestinationStore } from './destination';
+import { apiServerCaller } from '@services/apiCaller';
+import { config } from '@services/config';
 
 export interface IConnectionsStore {
   connections: ISourceConnections;
@@ -40,8 +42,8 @@ export class ConnectionsStore implements IConnectionsStore {
     this.loadAndSave();
   }
 
-  public loadAndSave() {
-    this.load();
+  public async loadAndSave() {
+    await this.load();
     autoSave(this, this.save.bind(this));
   }
 
@@ -51,9 +53,15 @@ export class ConnectionsStore implements IConnectionsStore {
     return connectionsStore;
   }
 
-  public load() {
-    const connectionsStore = localStorage.getItem('connectionsStore');
-    if (connectionsStore) {
+  public async load() {
+    let connectionsStore;
+    if (config.persistanceMode === 'file') {
+      const resp = await apiServerCaller().get('/loadConnections');
+      connectionsStore = resp.data.connections;
+    } else {
+      connectionsStore = localStorage.getItem('connectionsStore');
+    }
+    if (connectionsStore && connectionsStore !== '{}') {
       const store: IConnectionsStore = JSON.parse(connectionsStore);
       set(this, store);
     }
@@ -64,7 +72,11 @@ export class ConnectionsStore implements IConnectionsStore {
   }
 
   public save(json: string) {
-    localStorage.setItem('connectionsStore', json);
+    if (config.persistanceMode === 'file') {
+      apiServerCaller().post('/saveConnections', { connections: json });
+    } else {
+      localStorage.setItem('connectionsStore', json);
+    }
   }
 
   @action.bound