issue #22: refactor incoming_to_model, split out testing, split triggers

server/README.md

@@ -96,7 +96,7 @@ Because Heroku requires the app to be in the root, we use subtree to push:
 git subtree push --prefix server heroku master
 ```
 
-Updating schema:
+Creating/updating schema on Heroku instance:
 
 ```bash
 psql -v "ON_ERROR_STOP=1" -b -1 -e -f sql/PSQL.sql `heroku pg:credentials:url | tail -1`

server/sql/005-psql-create-extensions.sql

@@ -1,8 +1,4 @@
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
--- CREATE EXTENSION IF NOT EXISTS postgres_fdw;
 CREATE EXTENSION IF NOT EXISTS unaccent;
 CREATE EXTENSION IF NOT EXISTS pgcrypto;
-CREATE EXTENSION IF NOT EXISTS citext;
-
-CREATE EXTENSION IF NOT EXISTS citext;
--- CREATE EXTENSION IF NOT EXISTS pgtap;
+CREATE EXTENSION IF NOT EXISTS citext;

server/sql/008-psql-create-api.sql

@@ -2,4 +2,4 @@
 \ir api/schema.sql
 \ir api/snapshot.sql
 \ir api/warnings.sql
-\ir api/functions.sql
+\ir api/functions.sql

server/sql/010-psql-create-incoming.sql

@@ -18,7 +18,4 @@
 \ir incoming/entry_union.sql
 
 -- reports
-\ir incoming/warnings.sql
-
--- import facilities
-\ir incoming/update_from_server.sql
+\ir incoming/warnings.sql

server/sql/020-psql-create-model.sql

@@ -3,11 +3,15 @@
 \ir model/email.sql
 \ir model/audit.sql
 \ir model/model.sql
-\ir model/add_entry.sql
-\ir model/add_project_config.sql
 \ir model/ledger.sql
 \ir model/balance.sql
 
+\ir model/timesheet.sql
+\ir model/project_config.sql
+
+\ir model/add_entry.sql
+\ir model/add_project_config.sql
+
 \ir model/incoming_to_model.sql
 \ir model/incoming_to_model_transfer.sql
 \ir model/convert_all.sql

server/sql/022-psql-create-model-functions.sql

@@ -1 +1,2 @@
 \ir model/functions.sql
+\ir model/api.snapshot.trigger.sql

server/sql/050-psql-create-postgraphql.sql

@@ -1,3 +1,4 @@
 
 -- postgraphql interface
-\ir postgraphql/postgraphql.sql
+\ir postgraphql/postgraphql.sql
+\ir postgraphql/model.entry.trigger.sql

server/sql/500-psql-create-import-facilities-localhost.sql

@@ -0,0 +1,2 @@
+-- import facilities
+\ir api/update_from_server.sql

server/sql/900-psql-testsuite.sql

@@ -1,3 +1,4 @@
+\ir 003-psql-create-extensions-localhost.sql
 \ir test/schema.sql
 \ir test/api.sql
 \ir test/test.utils.sql

server/sql/api/functions.sql

@@ -1,7 +1,3 @@
-
-
-
-
 CREATE OR REPLACE FUNCTION api.snapshot(text, json)
 RETURNS SETOF api.warnings AS
 $func$
@@ -11,11 +7,12 @@ BEGIN
 	    ON CONFLICT(id) DO
 	      UPDATE SET doc = EXCLUDED.doc, ts = now() WHERE snapshot.id = EXCLUDED.id;
 	RETURN QUERY SELECT * FROM api.warnings WHERE id = $1;
-	IF NOT FOUND THEN
-		RETURN QUERY SELECT $1::text AS id, doc, table_name, error
-			FROM incoming.warnings
-			WHERE doc->>'project_id' = $1;
-	END IF;
+-- TODO: this times out, need better performance out of retrieving warnings
+--	IF NOT FOUND THEN
+--		RETURN QUERY SELECT $1::text AS id, doc, table_name, error
+--			FROM incoming.warnings
+--			WHERE doc->>'project_id' = $1;
+--	END IF;
 END;
 $func$  LANGUAGE plpgsql;
 

server/sql/incoming/update_from_server.sql → server/sql/api/update_from_server.sql

@@ -1,8 +1,8 @@
 -- Usage:
--- SELECT * FROM incoming.update_from_server(
+-- SELECT * FROM api.update_from_server(
 --	<OUTPUT FROM heroku pg:credentials:url>
 -- );
-CREATE OR REPLACE FUNCTION incoming.update_from_server(connection_url TEXT) RETURNS SETOF api.snapshot AS
+CREATE OR REPLACE FUNCTION api.update_from_server(connection_url TEXT) RETURNS SETOF api.snapshot AS
 $update_from_server$
 DECLARE
 	matches TEXT[];
@@ -13,11 +13,16 @@ DECLARE
 	password TEXT;
 	current_user TEXT;
 BEGIN
-	SELECT arr[1], arr[2], arr[3], arr[4], arr[5] INTO STRICT username, password, host, port, dbname FROM (
-		SELECT regexp_match(
-			connection_url,
-			'postgres:\/\/([a-zA-Z0-9]*):([a-zA-Z0-9]*)@([a-zA-Z0-9\-\.]*):(\d*)\/([a-zA-Z0-9]*)') AS arr
-	) t
+	SELECT 
+		arr[1], 
+		arr[2], 
+		arr[3], 
+		arr[4], 
+		arr[5] INTO STRICT username, password, host, port, dbname FROM (
+			SELECT regexp_match(
+				connection_url,
+				'postgres:\/\/([a-zA-Z0-9]*):([a-zA-Z0-9]*)@([a-zA-Z0-9\-\.]*):(\d*)\/([a-zA-Z0-9]*)') AS arr
+		) t
 	;
 
 	IF NOT FOUND THEN
@@ -38,18 +43,18 @@ BEGIN
 	        OPTIONS (user '%s', password '%s');
 	$$, current_user, username, password);
 
-	DROP FOREIGN TABLE IF EXISTS foreign_incoming_snapshot;
-	CREATE FOREIGN TABLE foreign_incoming_snapshot (
+	DROP FOREIGN TABLE IF EXISTS foreign_api_snapshot;
+	CREATE FOREIGN TABLE foreign_api_snapshot (
 		doc json NOT NULL,
 		ts timestamptz NOT NULL DEFAULT now(),
 		id text NOT NULL
 	)
 	SERVER foreign_server
-	OPTIONS (schema_name 'incoming', table_name 'snapshot');
+	OPTIONS (schema_name 'api', table_name 'snapshot');
 
 	RETURN QUERY INSERT INTO api.snapshot
-		SELECT remote.*
-			FROM foreign_incoming_snapshot remote
+		SELECT remote.id, remote.doc, remote.ts
+			FROM foreign_api_snapshot remote
 				LEFT JOIN api.snapshot ON (snapshot.id = remote.id)
 			WHERE snapshot.ts IS NULL OR (snapshot.ts < remote.ts)
 	ON CONFLICT(id)

server/sql/incoming/people.sql

@@ -13,7 +13,7 @@ CREATE OR REPLACE VIEW incoming.people_project AS
 		incoming.extract_percentage(doc->>'ratediscount')  AS project_rate_discount,
 		incoming.extract_rate(doc->>'rate') AS project_rate,
 		incoming.extract_currency(doc->>'rate') AS currency,
-		NULLIF(trim(doc->>'calendar'), '') AS calendar
+		NULLIF(trim(doc->>'calendarid'), '') AS calendar
 	FROM incoming.raw_people
 	;
 
@@ -41,7 +41,7 @@ CREATE OR REPLACE VIEW incoming.people AS
 		altnames,
 		nicknames
 	FROM
-		incoming.profile LEFT JOIN LATERAL (
+		incoming.profile INNER JOIN LATERAL (
 			SELECT array_agg(resource) AS nicknames
 			FROM incoming.nickname_to_email
 			WHERE nickname_to_email.email = profile.email
@@ -53,7 +53,7 @@ CREATE OR REPLACE VIEW incoming.people_project_calendar AS
 		resource,
 		email,
 		project_id,
-		(regexp_match(calendar, '.*src=([[a-z0-9\.]*)'))[1] || '@group.calendar.google.com'  AS calendar_id
+		(regexp_match(calendar, '([[a-z0-9\.]*@group\.calendar\.google\.com)'))[1]  AS calendar_id
 	FROM incoming.people_project
 	WHERE calendar IS NOT NULL
 	;

server/sql/model/api.snapshot.trigger.sql

@@ -0,0 +1,19 @@
+CREATE OR REPLACE FUNCTION model.convert_incoming_to_model_trigger() RETURNS trigger AS
+$convert_incoming_to_model_trigger$
+BEGIN
+
+	PERFORM  * FROM api.warnings WHERE id = NEW.id;
+	IF FOUND THEN
+		RETURN NEW;
+	END IF;
+	PERFORM model.convert_incoming_to_model(NEW.id);
+	RETURN NEW;
+END;
+$convert_incoming_to_model_trigger$ LANGUAGE PLPGSQL;
+
+DROP TRIGGER IF EXISTS model_update ON api.snapshot;
+
+CREATE TRIGGER model_update
+    AFTER INSERT OR UPDATE ON api.snapshot
+    FOR EACH ROW
+    EXECUTE PROCEDURE model.convert_incoming_to_model_trigger();

server/sql/model/convert_all.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION model.convert_all() RETURNS SETOF model.entry AS
+CREATE OR REPLACE FUNCTION model.convert_all() RETURNS SETOF uuid AS
 $$
     SELECT model.convert_incoming_to_model((s.id)::TEXT) FROM (
         SELECT id FROM api.snapshot

server/sql/model/incoming_to_model.sql

@@ -1,5 +1,5 @@
-CREATE OR REPLACE FUNCTION model.convert_incoming_to_model(_id TEXT) RETURNS SETOF model.entry AS
-$convert_incoming_to_model$
+CREATE OR REPLACE FUNCTION model.convert_organization(_id TEXT) RETURNS SETOF model.organization AS
+$convert_organization$
 	INSERT INTO model.organization(name, properties) 
 		SELECT * FROM (
 			WITH properties AS (
@@ -22,8 +22,12 @@ $convert_incoming_to_model$
 		) converted
 	ON CONFLICT(name)
 		DO UPDATE SET properties = EXCLUDED.properties WHERE organization.properties != EXCLUDED.properties
-		;
-
+	RETURNING *
+	;
+$convert_organization$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_person(_id TEXT) RETURNS SETOF model.person AS
+$convert_person$
 	INSERT INTO model.person(name, email, properties) SELECT * FROM (
 		WITH properties AS (
 			SELECT
@@ -55,15 +59,19 @@ $convert_incoming_to_model$
 			FROM properties
 				LEFT JOIN LATERAL UNNEST(properties.names, properties.values) AS p(pname, pvalue)
 				ON TRUE
-			WHERE pvalue IS NOT null
+			WHERE pvalue IS NOT NULL AND email IS NOT NULL
 			GROUP BY name, email
 	) converted
 	ON CONFLICT(email)
 		DO UPDATE SET properties = EXCLUDED.properties
 		WHERE person.email = EXCLUDED.email
 		AND person.properties != EXCLUDED.properties
+	RETURNING *
 	;
-
+$convert_person$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_account(_id TEXT) RETURNS SETOF model.account AS
+$convert_account$
 	WITH vendor AS (
 		SELECT id
 			FROM model.organization WHERE name = 'Coderbunker Shanghai'
@@ -99,8 +107,12 @@ $convert_incoming_to_model$
 		DO UPDATE SET properties = EXCLUDED.properties
 		WHERE account.name = EXCLUDED.name
 		AND account.properties != EXCLUDED.properties
+	RETURNING *
 	;
-
+$convert_account$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_project(_id TEXT) RETURNS SETOF model.project AS
+$convert_project$
 	INSERT INTO model.project(name, properties, account_id) SELECT * FROM (
 		WITH properties AS (
 			SELECT
@@ -128,8 +140,12 @@ $convert_incoming_to_model$
 		DO UPDATE SET properties = EXCLUDED.properties
 		WHERE project.properties->>'docid' = EXCLUDED.properties->>'docid'
 		AND project.properties != EXCLUDED.properties
+	RETURNING *
 	;
-
+$convert_project$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_membership(_id TEXT) RETURNS SETOF model.membership AS
+$convert_membership$
 	INSERT INTO model.membership(project_id, person_id, name, properties) SELECT * FROM (
 		WITH properties AS (
 			SELECT
@@ -160,8 +176,12 @@ $convert_incoming_to_model$
 	) converted
 	ON CONFLICT(project_id, name)
 		DO NOTHING
+	RETURNING *
 	;
-
+$convert_membership$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_rate(_id TEXT) RETURNS SETOF model.rate AS
+$convert_rate$
 	INSERT INTO model.rate(membership_id, rate, discount, currency, basis, valid) SELECT * FROM (
 		WITH project_rate_validity AS (
 			-- TODO: need to manage validity period
@@ -194,8 +214,12 @@ $convert_incoming_to_model$
 		WHERE rate.membership_id = EXCLUDED.membership_id
 			AND rate.basis = EXCLUDED.basis
 			AND rate.discount != EXCLUDED.discount
+	RETURNING *
 	;
-
+$convert_rate$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_task(_id TEXT) RETURNS SETOF model.task AS
+$convert_task$
 	INSERT INTO model.task(project_id, name) SELECT * FROM (
 		WITH tasks AS (
 			SELECT
@@ -219,8 +243,13 @@ $convert_incoming_to_model$
 	) converted
 	ON CONFLICT(project_id, name)
 		DO NOTHING
+	RETURNING *
 	;
 
+$convert_task$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION model.convert_entry(_id TEXT) RETURNS SETOF model.entry AS
+$convert_entry$
 	WITH incoming_timesheet AS (
 		SELECT entry.*
 			FROM incoming.entry_union AS entry
@@ -254,25 +283,18 @@ $convert_incoming_to_model$
 	ON CONFLICT(membership_id, start_datetime, stop_datetime)
 		DO NOTHING
 	RETURNING *
-;
-$convert_incoming_to_model$ LANGUAGE SQL;
-
-CREATE OR REPLACE FUNCTION model.convert_incoming_to_model_trigger() RETURNS trigger AS
-$convert_incoming_to_model_trigger$
-BEGIN
-
-	PERFORM  * FROM api.warnings WHERE id = NEW.id;
-	IF FOUND THEN
-		RETURN NEW;
-	END IF;
-	PERFORM model.convert_incoming_to_model(NEW.id);
-	RETURN NEW;
-END;
-$convert_incoming_to_model_trigger$ LANGUAGE PLPGSQL;
-
-DROP TRIGGER IF EXISTS model_update ON api.snapshot;
+	;
+$convert_entry$ LANGUAGE SQL;
 
-CREATE TRIGGER model_update
-    AFTER INSERT OR UPDATE ON api.snapshot
-    FOR EACH ROW
-    EXECUTE PROCEDURE model.convert_incoming_to_model_trigger();
+CREATE OR REPLACE FUNCTION model.convert_incoming_to_model(_id TEXT) RETURNS SETOF uuid AS
+$convert_incoming_to_model$
+	SELECT id FROM model.convert_organization(_id);
+	SELECT id FROM model.convert_person(_id);
+	SELECT id FROM model.convert_account(_id);
+	SELECT id FROM model.convert_project(_id);
+	SELECT id FROM model.convert_membership(_id);
+	SELECT id FROM model.convert_rate(_id);
+	SELECT id FROM model.convert_task(_id);
+	SELECT id FROM model.convert_entry(_id);
+;
+$convert_incoming_to_model$ LANGUAGE SQL;