Merge pull request #23 from coderbunker/issue22-release-20180504

issue #22: fixes for deployment to Heroku

server/README.md

@@ -96,26 +96,34 @@ Because Heroku requires the app to be in the root, we use subtree to push:
 git subtree push --prefix server heroku master
 ```
 
+Creating/updating schema on Heroku instance:
+
+```bash
+psql -v "ON_ERROR_STOP=1" -b -1 -e -f sql/PSQL.sql `heroku pg:credentials:url | tail -1`
+```
+
+Restarting the dyno (to load changes to the database for example)
+
+```bash
+heroku restart -a coderbunker-timesheet
+```
+
+## data transfer to/from heroku
+
 Pushing the local database:
 
 ```bash
 heroku pg:push timesheet postgresql-rigid-65921 --app coderbunker-timesheet
 ```
 
-Puling the Heroku database locally and making a copy before changing the pulled version
+Pulling the Heroku database locally and making a copy before changing the pulled version
 (adjust date):
 
 ```bash
 heroku pg:pull postgresql-rigid-65921 heroku-timesheet --app coderbunker-timesheet
 psql -c 'CREATE DATABASE "heroku-timesheet-20180416" TEMPLATE "heroku-timesheet";' postgres
 ```
 
-
-Restarting the dyno (to load changes to the database for example)
-
-```bash
-heroku restart -a coderbunker-timesheet
-```
 ## Manage Domain
 
 ### CNAME Setup for Heroku app

server/sql/003-psql-create-extensions-localhost.sql

@@ -0,0 +1,2 @@
+CREATE EXTENSION IF NOT EXISTS pgtap;
+CREATE EXTENSION IF NOT EXISTS postgres_fdw;

server/sql/005-psql-create-extensions.sql

@@ -1,8 +1,4 @@
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-CREATE EXTENSION IF NOT EXISTS postgres_fdw;
 CREATE EXTENSION IF NOT EXISTS unaccent;
 CREATE EXTENSION IF NOT EXISTS pgcrypto;
-CREATE EXTENSION IF NOT EXISTS citext;
-
-CREATE EXTENSION IF NOT EXISTS citext;
-CREATE EXTENSION IF NOT EXISTS pgtap;
+CREATE EXTENSION IF NOT EXISTS citext;

server/sql/008-psql-create-api.sql

@@ -2,4 +2,4 @@
 \ir api/schema.sql
 \ir api/snapshot.sql
 \ir api/warnings.sql
-\ir api/functions.sql
+\ir api/functions.sql

server/sql/010-psql-create-incoming.sql

@@ -18,7 +18,4 @@
 \ir incoming/entry_union.sql
 
 -- reports
-\ir incoming/warnings.sql
-
--- import facilities
-\ir incoming/update_from_server.sql
+\ir incoming/warnings.sql

server/sql/020-psql-create-model.sql

@@ -3,11 +3,16 @@
 \ir model/email.sql
 \ir model/audit.sql
 \ir model/model.sql
-\ir model/add_entry.sql
-\ir model/add_project_config.sql
+\ir model/symbol_to_currency.sql
 \ir model/ledger.sql
 \ir model/balance.sql
 
+\ir model/timesheet.sql
+\ir model/project_config.sql
+
+\ir model/add_entry.sql
+\ir model/add_project_config.sql
+
 \ir model/incoming_to_model.sql
 \ir model/incoming_to_model_transfer.sql
 \ir model/convert_all.sql

server/sql/022-psql-create-model-functions.sql

@@ -1 +1,2 @@
 \ir model/functions.sql
+\ir model/api.snapshot.trigger.sql

server/sql/025-psql-create-model-data.sql

@@ -1,10 +1,11 @@
 
-DO $$ 
+DO $$
  BEGIN
 	INSERT INTO model.organization(id, name, properties)
-		VALUES 
-			('46207d44-ddf3-4ecf-8c01-d88d56d56181', 'Coderbunker Shanghai', '{}'), 
-			('dffae778-dd06-46c1-a4ee-b7bfce34f71d', 'Coderbunker Singapore', '{}')
+		VALUES
+			('46207d44-ddf3-4ecf-8c01-d88d56d56181', 'Coderbunker Shanghai', '{}'),
+			('dffae778-dd06-46c1-a4ee-b7bfce34f71d', 'Coderbunker Singapore', '{}'),
+			('3bbc0fdb-8e2a-40a1-b0e6-3e1fc4ca66b2', 'Agora Space', '{}')
 	ON CONFLICT DO NOTHING;
  END;
 $$;

server/sql/040-psql-create-reports.sql

@@ -1,2 +1,3 @@
 \ir report/report.sql
-\ir report/report.missing_receivable_deposit.sql
+\ir report/report.missing_receivable_deposit.sql
+\ir report/report.time_tracking_matrix.sql

server/sql/050-psql-create-postgraphql.sql

@@ -1,3 +1,4 @@
 
 -- postgraphql interface
-\ir postgraphql.sql
+\ir postgraphql/postgraphql.sql
+\ir postgraphql/model.entry.trigger.sql

server/sql/500-psql-create-import-facilities-localhost.sql

@@ -0,0 +1,2 @@
+-- import facilities
+\ir api/update_from_server.sql

server/sql/900-psql-testsuite.sql

@@ -1,3 +1,4 @@
+\ir 003-psql-create-extensions-localhost.sql
 \ir test/schema.sql
 \ir test/api.sql
 \ir test/test.utils.sql

server/sql/PSQL.sql

@@ -9,4 +9,6 @@
 \ir 030-psql-incoming-to-model.sql
 \ir 040-psql-create-reports.sql
 \ir 050-psql-create-postgraphql.sql
-\ir 900-psql-testsuite.sql
+-- \ir 900-psql-testsuite.sql
+
+\echo "Success. Please don't forget to run tests using watch-test.sh"

server/sql/api/functions.sql

@@ -1,7 +1,3 @@
-
-
-
-
 CREATE OR REPLACE FUNCTION api.snapshot(text, json)
 RETURNS SETOF api.warnings AS
 $func$
@@ -11,11 +7,12 @@ BEGIN
 	    ON CONFLICT(id) DO
 	      UPDATE SET doc = EXCLUDED.doc, ts = now() WHERE snapshot.id = EXCLUDED.id;
 	RETURN QUERY SELECT * FROM api.warnings WHERE id = $1;
-	IF NOT FOUND THEN
-		RETURN QUERY SELECT $1::text AS id, doc, table_name, error
-			FROM incoming.warnings
-			WHERE doc->>'project_id' = $1;
-	END IF;
+-- TODO: this times out, need better performance out of retrieving warnings
+--	IF NOT FOUND THEN
+--		RETURN QUERY SELECT $1::text AS id, doc, table_name, error
+--			FROM incoming.warnings
+--			WHERE doc->>'project_id' = $1;
+--	END IF;
 END;
 $func$  LANGUAGE plpgsql;
 

server/sql/incoming/update_from_server.sql → server/sql/api/update_from_server.sql

@@ -1,8 +1,8 @@
 -- Usage:
--- SELECT * FROM incoming.update_from_server(
+-- SELECT * FROM api.update_from_server(
 --	<OUTPUT FROM heroku pg:credentials:url>
 -- );
-CREATE OR REPLACE FUNCTION incoming.update_from_server(connection_url TEXT) RETURNS SETOF api.snapshot AS
+CREATE OR REPLACE FUNCTION api.update_from_server(connection_url TEXT) RETURNS SETOF api.snapshot AS
 $update_from_server$
 DECLARE
 	matches TEXT[];
@@ -13,11 +13,16 @@ DECLARE
 	password TEXT;
 	current_user TEXT;
 BEGIN
-	SELECT arr[1], arr[2], arr[3], arr[4], arr[5] INTO STRICT username, password, host, port, dbname FROM (
-		SELECT regexp_match(
-			connection_url,
-			'postgres:\/\/([a-zA-Z0-9]*):([a-zA-Z0-9]*)@([a-zA-Z0-9\-\.]*):(\d*)\/([a-zA-Z0-9]*)') AS arr
-	) t
+	SELECT 
+		arr[1], 
+		arr[2], 
+		arr[3], 
+		arr[4], 
+		arr[5] INTO STRICT username, password, host, port, dbname FROM (
+			SELECT regexp_match(
+				connection_url,
+				'postgres:\/\/([a-zA-Z0-9]*):([a-zA-Z0-9]*)@([a-zA-Z0-9\-\.]*):(\d*)\/([a-zA-Z0-9]*)') AS arr
+		) t
 	;
 
 	IF NOT FOUND THEN
@@ -38,18 +43,18 @@ BEGIN
 	        OPTIONS (user '%s', password '%s');
 	$$, current_user, username, password);
 
-	DROP FOREIGN TABLE IF EXISTS foreign_incoming_snapshot;
-	CREATE FOREIGN TABLE foreign_incoming_snapshot (
+	DROP FOREIGN TABLE IF EXISTS foreign_api_snapshot;
+	CREATE FOREIGN TABLE foreign_api_snapshot (
 		doc json NOT NULL,
 		ts timestamptz NOT NULL DEFAULT now(),
 		id text NOT NULL
 	)
 	SERVER foreign_server
-	OPTIONS (schema_name 'incoming', table_name 'snapshot');
+	OPTIONS (schema_name 'api', table_name 'snapshot');
 
 	RETURN QUERY INSERT INTO api.snapshot
-		SELECT remote.*
-			FROM foreign_incoming_snapshot remote
+		SELECT remote.id, remote.doc, remote.ts
+			FROM foreign_api_snapshot remote
 				LEFT JOIN api.snapshot ON (snapshot.id = remote.id)
 			WHERE snapshot.ts IS NULL OR (snapshot.ts < remote.ts)
 	ON CONFLICT(id)

server/sql/incoming/people.sql

@@ -13,16 +13,25 @@ CREATE OR REPLACE VIEW incoming.people_project AS
 		incoming.extract_percentage(doc->>'ratediscount')  AS project_rate_discount,
 		incoming.extract_rate(doc->>'rate') AS project_rate,
 		incoming.extract_currency(doc->>'rate') AS currency,
-		NULLIF(trim(doc->>'calendar'), '') AS calendar
+		NULLIF(trim(doc->>'calendarid'), '') AS calendar
 	FROM incoming.raw_people
 	;
 
-DROP MATERIALIZED VIEW IF EXISTS incoming.nickname_to_email CASCADE;
-CREATE MATERIALIZED VIEW incoming.nickname_to_email AS
-	SELECT resource, email
-	FROM incoming.people_project
-	WHERE resource IS NOT NULL
-	GROUP BY resource, email;
+DO $$
+	BEGIN
+		PERFORM * FROM pg_catalog.pg_matviews WHERE matviewname = 'nickname_to_email' AND schemaname = 'incoming';
+		IF NOT FOUND THEN
+			CREATE MATERIALIZED VIEW incoming.nickname_to_email AS
+				SELECT resource, email
+				FROM incoming.people_project
+				WHERE resource IS NOT NULL
+				GROUP BY resource, email;
+			CREATE UNIQUE INDEX nickname_to_email_index ON incoming.nickname_to_email(resource, email);
+		ELSE
+			REFRESH MATERIALIZED VIEW CONCURRENTLY incoming.nickname_to_email;
+		END IF;
+	END;
+$$ LANGUAGE PLPGSQL;
 
 CREATE OR REPLACE VIEW incoming.people AS
 	SELECT
@@ -32,7 +41,7 @@ CREATE OR REPLACE VIEW incoming.people AS
 		altnames,
 		nicknames
 	FROM
-		incoming.profile LEFT JOIN LATERAL (
+		incoming.profile INNER JOIN LATERAL (
 			SELECT array_agg(resource) AS nicknames
 			FROM incoming.nickname_to_email
 			WHERE nickname_to_email.email = profile.email
@@ -44,7 +53,7 @@ CREATE OR REPLACE VIEW incoming.people_project_calendar AS
 		resource,
 		email,
 		project_id,
-		(regexp_match(calendar, '.*src=([[a-z0-9\.]*)'))[1] || '@group.calendar.google.com'  AS calendar_id
+		(regexp_match(calendar, '([[a-z0-9\.]*@group\.calendar\.google\.com)'))[1]  AS calendar_id
 	FROM incoming.people_project
 	WHERE calendar IS NOT NULL
 	;

server/sql/model/api.snapshot.trigger.sql

@@ -0,0 +1,19 @@
+CREATE OR REPLACE FUNCTION model.convert_incoming_to_model_trigger() RETURNS trigger AS
+$convert_incoming_to_model_trigger$
+BEGIN
+
+	PERFORM  * FROM api.warnings WHERE id = NEW.id;
+	IF FOUND THEN
+		RETURN NEW;
+	END IF;
+	PERFORM model.convert_incoming_to_model(NEW.id);
+	RETURN NEW;
+END;
+$convert_incoming_to_model_trigger$ LANGUAGE PLPGSQL;
+
+DROP TRIGGER IF EXISTS model_update ON api.snapshot;
+
+CREATE TRIGGER model_update
+    AFTER INSERT OR UPDATE ON api.snapshot
+    FOR EACH ROW
+    EXECUTE PROCEDURE model.convert_incoming_to_model_trigger();