From 9137446ce281b5fe4ff276cc83daa17e41fad1c2 Mon Sep 17 00:00:00 2001 From: Ricky Ng-Adam Date: Sat, 11 Aug 2018 14:31:48 +0800 Subject: [PATCH 1/4] issue #40: create internal schema --- GC.md => GOOGLE_CLOUD.md | 0 HEROKU.md | 61 ++++++++++++++ INSTALL.md | 98 ++-------------------- app.js | 140 +++++++++++++++++++++++++------ db/index.js | 10 --- package.json | 9 +- routes/index.js | 5 -- routes/snapshot.js | 37 -------- sql/070-psql-create-internal.sql | 4 + sql/PSQL.sql | 1 + sql/api/update_from_server.sql | 12 +-- sql/incoming/group.sql | 43 ++++++++++ sql/internal/group.sql | 19 +++++ sql/internal/schema.sql | 1 + test/internal_test.js | 86 +++++++++++++++++++ 15 files changed, 350 insertions(+), 176 deletions(-) rename GC.md => GOOGLE_CLOUD.md (100%) create mode 100644 HEROKU.md delete mode 100644 db/index.js delete mode 100644 routes/index.js delete mode 100644 routes/snapshot.js create mode 100644 sql/070-psql-create-internal.sql create mode 100644 sql/incoming/group.sql create mode 100644 sql/internal/group.sql create mode 100644 sql/internal/schema.sql create mode 100644 test/internal_test.js diff --git a/GC.md b/GOOGLE_CLOUD.md similarity index 100% rename from GC.md rename to GOOGLE_CLOUD.md diff --git a/HEROKU.md b/HEROKU.md new file mode 100644 index 0000000..f8a6cd2 --- /dev/null +++ b/HEROKU.md @@ -0,0 +1,61 @@ +# Heroku Deployment + +## Manage Domain + +### CNAME Setup for Heroku app + +1. Get CNAME from heroku: `heroku domains -a coderbunker-timesheet` + +2. add CNAME to google domains + + | NAME | TYPE | TTL | DATA | + |--------|:--------------:|------:|--------------------------------------:| + | data | CNAME | 1h | data.coderbunker.com.herokudns.com. | + + +### SSL Setup + +Enable SSL automatically managed by heroku. + +## troubleshooting + +want to push an amended history with subtree push? sadly, does not support push. + +create a local branch and force push that first: + +``` +git subtree split --prefix server -b backup-branch +git push -f heroku backup-branch:master +``` + +should now be back to normal... + +## deploying to Heroku + +Creating/updating schema on Heroku instance: + +```bash +psql -v "ON_ERROR_STOP=1" -b -1 -e -f sql/PSQL.sql `heroku pg:credentials:url | tail -1` +``` + +Restarting the dyno (to load changes to the database for example) + +```bash +heroku restart -a coderbunker-timesheet +``` + +## data transfer to/from heroku database + +Pushing the local database: + +```bash +heroku pg:push timesheet postgresql-rigid-65921 --app coderbunker-timesheet +``` + +Pulling the Heroku database locally and making a copy before changing the pulled version +(adjust date): + +```bash +heroku pg:pull postgresql-rigid-65921 heroku-timesheet --app coderbunker-timesheet +psql -c 'CREATE DATABASE "heroku-timesheet-20180416" TEMPLATE "heroku-timesheet";' postgres +``` diff --git a/INSTALL.md b/INSTALL.md index a3a1454..8a31998 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,4 +1,4 @@ -# Integration with Google Spreadsheet +# Running the system locally ## setup NodeJS latest LTS @@ -12,7 +12,7 @@ nvm install --lts ```bash npm install -node app.js +npm start ``` install PostgreSQL extensions: @@ -20,6 +20,7 @@ install PostgreSQL extensions: ```bash pgxn install pgtap ``` + create schema of DB ```bash @@ -35,28 +36,9 @@ this will also run the test suite. ./watch-test.sh timesheet ``` -## Apache hook - -Apache config: - -```text - - ProxyPass "http://localhost:3000/spreadsheet" - -``` - -restart: - -```bash -systemctl restart apache2 -``` - -## Setup +## Exposing local as a web service -creates two routes: - -* /spreadsheet/snapshot/SPREADSHEET_ID -* /spreadsheet/change/SPREADSHEET_ID +Use http://ngrok.io ## Testing @@ -69,7 +51,7 @@ Sample file: "apptype": "Spreadsheet", "category": "Timesheet" }, - "apikey": "f98dbe87-5749-47c6-8e39-47ae7ff401ac" + "apikey": "STRING_CONFIGURED_IN_ENV" } ``` @@ -91,70 +73,4 @@ database: ```bash pg_dump -N postgraphql_watch -O -s postgresql://localhost/timesheet > sql/timesheet.sql -``` - -## deploying to Heroku - -Because Heroku requires the app to be in the root, we use subtree to push: - -```bash -git subtree push --prefix server heroku master -``` - -Creating/updating schema on Heroku instance: - -```bash -psql -v "ON_ERROR_STOP=1" -b -1 -e -f sql/PSQL.sql `heroku pg:credentials:url | tail -1` -``` - -Restarting the dyno (to load changes to the database for example) - -```bash -heroku restart -a coderbunker-timesheet -``` - -## data transfer to/from heroku - -Pushing the local database: - -```bash -heroku pg:push timesheet postgresql-rigid-65921 --app coderbunker-timesheet -``` - -Pulling the Heroku database locally and making a copy before changing the pulled version -(adjust date): - -```bash -heroku pg:pull postgresql-rigid-65921 heroku-timesheet --app coderbunker-timesheet -psql -c 'CREATE DATABASE "heroku-timesheet-20180416" TEMPLATE "heroku-timesheet";' postgres -``` - -## Manage Domain - -### CNAME Setup for Heroku app - -1. Get CNAME from heroku: `heroku domains -a coderbunker-timesheet` - -2. add CNAME to google domains - - | NAME | TYPE | TTL | DATA | - |--------|:--------------:|------:|--------------------------------------:| - | data | CNAME | 1h | data.coderbunker.com.herokudns.com. | - - -### SSL Setup - -Enable SSL automatically managed by heroku. - -## troubleshooting - -want to push an amended history with subtree push? sadly, does not support push. - -create a local branch and force push that first: - -``` -git subtree split --prefix server -b backup-branch -git push -f heroku backup-branch:master -``` - -should now be back to normal... +``` \ No newline at end of file diff --git a/app.js b/app.js index 81371b3..efefc25 100644 --- a/app.js +++ b/app.js @@ -1,42 +1,130 @@ require('dotenv').config() const express = require('express') -const mountRoutes = require('./routes') +const Router = require('express-promise-router') const bodyParser = require('body-parser') const { postgraphile } = require("postgraphile"); +const cookieParser = require('cookie-parser'); +const { Pool } = require('pg') -const app = express() +function createQuery(dburl) { + console.log('connecting to DATABASE_URL=%s', dburl); -app.use(postgraphile( - process.env.DATABASE_URL || "postgres://localhost/heroku-timesheet", - "postgraphql", - { + const pool = new Pool({ + connectionString: dburl + }); + + return function(text, params) { + pool.query(text, params); + } +} + +function postgraphileDefaultConfig(config) { + config = config || {}; + return Object.assign({ dynamicJson: true, - disableDefaultMutations: true, + disableDefaultMutations: false, graphiql: true, watchPg: true, - enableCors: true + enableCors: true, + extendedErrors: ['hint', 'detail', 'errcode'] + }, config); +} + +function validateApiKey(apikey) { + return function(req, res, next) { + function err(e) { + res.writeHead(400); + res.end(JSON.stringify({error: e})); + } + if(req.originalUrl.match(/internal/)) { + if(!((req.body && req.body.apikey === apikey) || req.cookies.apikey === apikey)) { + return err(`invalid api key for ${req.originalUrl}`); + } + } + next(); } -)); +} + +// create a new express-promise-router +// this has the same API as the normal express router except +// it allows you to use async functions as route handlers +function createRouter(router, query) { + router.post('/snapshot', async (req, res) => { + function err(e) { + res.writeHead(400); + res.end(JSON.stringify({error: e})); + } + + if(!req.body.id) { + return err("id is not defined in body") + } + if(!req.body.doc) { + return err("doc is not defined in body") + } + const { rows, fields } = await query( + `SELECT api.snapshot_json($1, $2::json)`, + [ + req.body.id, + JSON.stringify(req.body.doc) + ]) + var json = JSON.stringify(rows[0].snapshot_json); + res.writeHead(200, {'content-type':'application/json', 'content-length': Buffer.byteLength(json)}); + res.end(json); + }); + + return router; +} + +function registerEndpoint(app, schema, endpoints) { + if(!endpoints) { + endpoints = { + graphqlRoute: `/${schema}/graphql`, + graphiqlRoute: `/${schema}/graphiql` + } + } + + app.use(postgraphile( + process.env.DATABASE_URL || "postgres://localhost/heroku-timesheet", + schema, + postgraphileDefaultConfig(endpoints) + )); +} + +function createServer(app, router, mountPoint, apikey, port) { + app.use(mountPoint, router); + app.use(cookieParser()); + app.use(validateApiKey(apikey)); + + registerEndpoint(app, 'postgraphql', {}); + registerEndpoint(app, 'internal'); + + // parse application/x-www-form-urlencoded + app.use(bodyParser.urlencoded({ + extended: false + })) -// parse application/x-www-form-urlencoded -app.use(bodyParser.urlencoded({ - extended: false -})) + // parse application/json + app.use(bodyParser.json({ + limit: "5mb" + })) -// parse application/json -app.use(bodyParser.json({ - limit: "5mb" -})) + app.use(express.static(__dirname + '/public')); -app.use(express.static(__dirname + '/public')); + var server = app.listen(port, function() { + console.log(JSON.stringify(server.address())) + const host = server.address().address; + const port = server.address().port; + console.log('timesheet app listening at http://%s:%s', host, port); + }); -mountRoutes(app) + return server; +} -console.log('port: %s', process.env.PORT) -var server = app.listen(process.env.PORT, () => { - console.log(JSON.stringify(server.address())) - const host = server.address().address; - const port = server.address().port; - console.log('timesheet app listening at http://%s:%s', host, port); -}); +if (require.main === module) { + const query = createQuery(process.env.DATABASE_URL); + const router = createRouter(new Router(), query); + const server = createServer(express(), router, '/gsuite', process.env.APIKEY, process.env.PORT); +} else { + module.exports = { createServer, createRouter, createQuery }; +} \ No newline at end of file diff --git a/db/index.js b/db/index.js deleted file mode 100644 index e1588bc..0000000 --- a/db/index.js +++ /dev/null @@ -1,10 +0,0 @@ -const { Pool } = require('pg') - -console.log('connecting to DATABASE_URL=%s', process.env.DATABASE_URL) -const pool = new Pool({ - connectionString: process.env.DATABASE_URL -}); - -module.exports = { - query: (text, params) => pool.query(text, params) -} \ No newline at end of file diff --git a/package.json b/package.json index 4545e78..006a007 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,19 @@ }, "homepage": "https://github.com/coderbunker/timesheet-backend#readme", "dependencies": { + "base64url": "^3.0.0", "body-parser": "^1.18.2", + "buffer-equal-constant-time": "^1.0.1", + "cookie-parser": "^1.4.3", "dotenv": "^5.0.0", "express": "^4.16.2", "express-promise-router": "^3.0.1", + "find-free-port": "^2.0.0", "googleapis": "^26.0.1", "pg": "^7.4.1", - "postgraphile": "^4.0.0-beta.2" + "postgraphile": "^4.0.0-rc.3" + }, + "devDependencies": { + "isomorphic-fetch": "^2.2.1" } } diff --git a/routes/index.js b/routes/index.js deleted file mode 100644 index de4873f..0000000 --- a/routes/index.js +++ /dev/null @@ -1,5 +0,0 @@ -const snapshot = require('./snapshot') - -module.exports = (app) => { - app.use('/gsuite', snapshot) -} \ No newline at end of file diff --git a/routes/snapshot.js b/routes/snapshot.js deleted file mode 100644 index 45ef2fd..0000000 --- a/routes/snapshot.js +++ /dev/null @@ -1,37 +0,0 @@ -const Router = require('express-promise-router') - -const db = require('../db') - -// create a new express-promise-router -// this has the same API as the normal express router except -// it allows you to use async functions as route handlers -const router = new Router() - -// export our router to be mounted by the parent application -module.exports = router - -router.post('/snapshot', async (req, res) => { - function err(e) { - res.writeHead(400); - res.end(JSON.stringify({error: e})); - } - if(req.body.apikey !== process.env.APIKEY) { - return err("invalid api key") - } - if(!req.body.id) { - return err("id is not defined in body") - } - if(!req.body.doc) { - return err("doc is not defined in body") - } - - const { rows, fields } = await db.query( - `SELECT api.snapshot_json($1, $2::json)`, - [ - req.body.id, - JSON.stringify(req.body.doc) - ]) - var json = JSON.stringify(rows[0].snapshot_json); - res.writeHead(200, {'content-type':'application/json', 'content-length': Buffer.byteLength(json)}); - res.end(json); -}); \ No newline at end of file diff --git a/sql/070-psql-create-internal.sql b/sql/070-psql-create-internal.sql new file mode 100644 index 0000000..1f9d98a --- /dev/null +++ b/sql/070-psql-create-internal.sql @@ -0,0 +1,4 @@ + +-- internal interface +\ir internal/schema.sql +\ir internal/group.sql diff --git a/sql/PSQL.sql b/sql/PSQL.sql index 9348b72..8b4b182 100644 --- a/sql/PSQL.sql +++ b/sql/PSQL.sql @@ -9,6 +9,7 @@ \ir 030-psql-incoming-to-model.sql \ir 040-psql-create-reports.sql \ir 050-psql-create-postgraphql.sql +\ir 070-psql-create-internal.sql -- \ir 900-psql-testsuite.sql \echo "Success. Please don't forget to run tests using watch-test.sh" \ No newline at end of file diff --git a/sql/api/update_from_server.sql b/sql/api/update_from_server.sql index 5a24440..4ca372b 100644 --- a/sql/api/update_from_server.sql +++ b/sql/api/update_from_server.sql @@ -1,6 +1,6 @@ -- Usage: -- SELECT * FROM api.update_from_server( --- +-- -- ); CREATE OR REPLACE FUNCTION api.update_from_server(connection_url TEXT) RETURNS SETOF api.snapshot AS $update_from_server$ @@ -13,11 +13,11 @@ DECLARE password TEXT; current_user TEXT; BEGIN - SELECT - arr[1], - arr[2], - arr[3], - arr[4], + SELECT + arr[1], + arr[2], + arr[3], + arr[4], arr[5] INTO STRICT username, password, host, port, dbname FROM ( SELECT regexp_match( connection_url, diff --git a/sql/incoming/group.sql b/sql/incoming/group.sql new file mode 100644 index 0000000..bec69af --- /dev/null +++ b/sql/incoming/group.sql @@ -0,0 +1,43 @@ +CREATE OR REPLACE VIEW incoming."raw_group" AS + SELECT + doc->>'id' AS group_id, + (regexp_matches(doc->>'email', '(.*)@coderbunker.com'))[1] AS label, + doc->>'name' AS group_description, + doc->>'email' AS group_email, + k.keys AS members + FROM api.snapshot + LEFT JOIN LATERAL (SELECT ARRAY(SELECT * FROM jsonb_object_keys(snapshot.doc->'members')) AS keys) k ON true + WHERE + doc->>'apptype' = 'Groups' + AND doc->>'category' = 'Membership' + AND k.keys[1] <> 'undefined' + GROUP BY doc->>'name', doc->>'email', doc->>'id', k.keys, ts + ORDER BY label ASC + ; + + +CREATE OR REPLACE VIEW incoming.group AS + SELECT + COALESCE(profile.email, labeled.email) AS email, + labeled.labels, + profile.fullname, + profile.github, + profile.wechat, + COALESCE(regexp_split_to_array(lower(profile.status), ', +'), ARRAY[]::TEXT[]) AS status + FROM ( + SELECT + array_agg(label) AS labels, + email + FROM ( + SELECT + group_id, + label, + unnest(members) AS email + FROM incoming.raw_group + ORDER BY label + ) t + WHERE label NOT LIKE '%-client' + GROUP BY email + ) labeled + FULL JOIN incoming.profile ON profile.email = labeled.email + ; \ No newline at end of file diff --git a/sql/internal/group.sql b/sql/internal/group.sql new file mode 100644 index 0000000..f6d1222 --- /dev/null +++ b/sql/internal/group.sql @@ -0,0 +1,19 @@ +DO $$ + BEGIN + PERFORM * + FROM pg_catalog.pg_matviews + WHERE matviewname = 'group' AND + schemaname = 'internal'; + IF NOT FOUND THEN + CREATE MATERIALIZED VIEW internal.group AS + SELECT + "group".*, + now() AS last_refresh + FROM incoming.group + ; + CREATE UNIQUE INDEX internal_group_index ON internal.group(email); + ELSE + REFRESH MATERIALIZED VIEW CONCURRENTLY internal.group; + END IF; + END; +$$ LANGUAGE PLPGSQL; \ No newline at end of file diff --git a/sql/internal/schema.sql b/sql/internal/schema.sql new file mode 100644 index 0000000..9666a40 --- /dev/null +++ b/sql/internal/schema.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS internal; \ No newline at end of file diff --git a/test/internal_test.js b/test/internal_test.js new file mode 100644 index 0000000..3a838b1 --- /dev/null +++ b/test/internal_test.js @@ -0,0 +1,86 @@ +const assert = require('assert'); +const fp = require("find-free-port") + +const fetch = require('isomorphic-fetch'); + +const express = require('express'); +const Router = require('express-promise-router') + +const appUnderTest = require('../app.js'); + +const allGroupsQuery = + ` + { + allGroups { + nodes { + email + labels + fullname + github + wechat + status + lastRefresh + } + } + } + `; + +const properties = [ + 'email', + 'fullname', + 'github', + 'labels', + 'lastRefresh', + 'status', + 'wechat', +]; + +describe('internal schema', () => { + + beforeEach((done) => { + fp(3000).then(([freep]) => { + this.port = freep; + query = appUnderTest.createQuery('postgres://localhost/heroku-timesheet'); + router = appUnderTest.createRouter(new Router(), query); + server = appUnderTest.createServer( + express(), router, '/gsuite', '', freep); + done(); + }).catch((err)=>{ + done(err); + }); + }); + + it('can query all groups', (done) => { + fetch( `http://localhost:${this.port}/internal/graphql`, { + body: JSON.stringify({query: allGroupsQuery}), + method: 'POST', + headers: { + "Content-Type": "application/json; charset=utf-8", + } + }) + .then((response) => { + if (response.status >= 400) { + throw new Error(`Bad response from server: ${JSON.stringify(response)}`); + } + return response.json(); + }) + .then((result) => { + if(!result.data) { + throw new Error(`missing expected results with {data: {allGroups}}: ${Object.keys(result)}`); + } + if(!result.data.allGroups) { + throw new Error(`missing expected results with {data: {allGroups}}: ${Object.keys(result.data)}`); + } + //console.log(JSON.stringify(result.data.allGroups.nodes)); + var keys = Object.keys(result.data.allGroups.nodes[0]) + keys.sort(); + if(JSON.stringify(keys) != JSON.stringify(properties)) { + throw new Error(`key(s) missing ${keys}`); + } + done(); + }) + .catch((err) => { + done(err); + }); + }); +}); From 23c4a842546ca9829220da0b0259d914884b5edc Mon Sep 17 00:00:00 2001 From: Ricky Ng-Adam Date: Sat, 11 Aug 2018 15:24:09 +0800 Subject: [PATCH 2/4] issue #40: missing group in creating incoming --- sql/010-psql-create-incoming.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/010-psql-create-incoming.sql b/sql/010-psql-create-incoming.sql index 78b3d59..22e235f 100644 --- a/sql/010-psql-create-incoming.sql +++ b/sql/010-psql-create-incoming.sql @@ -10,6 +10,7 @@ \ir incoming/profile.sql \ir incoming/entry.sql \ir incoming/people.sql +\ir incoming/group.sql \ir incoming/transfer.sql \ir incoming/waveapps.sql From eefdc5ba1343b24205be1b4976b026df0460af83 Mon Sep 17 00:00:00 2001 From: Ilkka Veima Date: Sat, 1 Sep 2018 17:12:59 +0800 Subject: [PATCH 3/4] Add test for query with invalid API key. Add npm test script to package.json. Change internal_test.js to use database string from .env config. Add test apikey to .env config. --- package.json | 3 ++- test/internal_test.js | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 006a007..35beed2 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "First version of the timesheet backend", "main": "index.js", "scripts": { - "start": "node app.js" + "start": "node app.js", + "test": "mocha" }, "repository": { "type": "git", diff --git a/test/internal_test.js b/test/internal_test.js index 3a838b1..7234bf5 100644 --- a/test/internal_test.js +++ b/test/internal_test.js @@ -8,6 +8,9 @@ const Router = require('express-promise-router') const appUnderTest = require('../app.js'); +// Get database URL from config +require('dotenv').config(); + const allGroupsQuery = ` { @@ -40,10 +43,10 @@ describe('internal schema', () => { beforeEach((done) => { fp(3000).then(([freep]) => { this.port = freep; - query = appUnderTest.createQuery('postgres://localhost/heroku-timesheet'); + query = appUnderTest.createQuery(process.env.DATABASE_URL); router = appUnderTest.createRouter(new Router(), query); server = appUnderTest.createServer( - express(), router, '/gsuite', '', freep); + express(), router, '/gsuite', process.env.TESTAPIKEY, freep); done(); }).catch((err)=>{ done(err); @@ -56,7 +59,9 @@ describe('internal schema', () => { method: 'POST', headers: { "Content-Type": "application/json; charset=utf-8", + "Cookie": `apikey=${process.env.TESTAPIKEY}`, } + }) .then((response) => { if (response.status >= 400) { @@ -83,4 +88,33 @@ describe('internal schema', () => { done(err); }); }); + + it('Can NOT query with invalid apikey', (done) => { + fetch(`http://localhost:${this.port}/internal/graphql`, { + body: JSON.stringify({query: allGroupsQuery}), + method: 'POST', + headers: { + "Content-Type": "application/json; charset=utf-8", + "Cookie": "apikey=WRONG_APIKEY", + } + }) + .then((response) => { + if(response.status != 400) { + throw new Error("Status code should be 400: Bad request"); + } + + return response.json(); + }) + .then((result) => { + const expected = { 'error': 'invalid api key for /internal/graphql'}; + if(result.error == expected.error) { + done(); + } + else + { + throw new Error(`Response should be ${JSON.stringify(expected, null, 4)}`); + } + }) + .catch((err) => { done(err) }); + }); }); From 6539e84335bbf350e5e9d1fbbd321676c6360562 Mon Sep 17 00:00:00 2001 From: Ilkka Veima Date: Sat, 1 Sep 2018 17:19:33 +0800 Subject: [PATCH 4/4] Add TESTAPIKEY to .env-template --- .env-template | 1 + 1 file changed, 1 insertion(+) diff --git a/.env-template b/.env-template index bb1d803..d7fc23a 100644 --- a/.env-template +++ b/.env-template @@ -4,3 +4,4 @@ PGPASSWORD= PGDATABASE=timesheet PGPORT=5432 PORT=3000 +TESTAPIKEY=secret