Merge branch 'master' into as_500

# Conflicts:
#	projects/or_scs/src/Mirza/SupplyChain/PopulateUtils.hs

deploy/docker-compose.yml

@@ -21,6 +21,38 @@ services:
         max-size: "200k"
         max-file: "10"
 
+  sci054edapi:
+    image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/edapi:latest
+    restart: always
+    env_file:
+      - sci054.env
+    depends_on:
+      - db
+      - sci054scs
+    networks:
+      - backend
+    logging:
+      driver: json-file
+      options:
+        max-size: "200k"
+        max-file: "10"
+
+  sci054scs:
+    image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/supplychainserver:latest
+    restart: always
+    env_file:
+      - sci054.env
+    depends_on:
+      - db
+    networks:
+      - backend
+    command: --orhost or --orport 8200 --init-db -c "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci054supplychainserver"
+    logging:
+      driver: json-file
+      options:
+        max-size: "200k"
+        max-file: "10"
+
   sci041edapi:
     image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/edapi:latest
     restart: always
@@ -53,6 +85,38 @@ services:
         max-size: "200k"
         max-file: "10"
 
+  sci042edapi:
+    image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/edapi:latest
+    restart: always
+    env_file:
+      - sci042.env
+    depends_on:
+      - db
+      - sci042scs
+    networks:
+      - backend
+    logging:
+      driver: json-file
+      options:
+        max-size: "200k"
+        max-file: "10"
+
+  sci042scs:
+    image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/supplychainserver:latest
+    restart: always
+    env_file:
+      - sci042.env
+    depends_on:
+      - db
+    networks:
+      - backend
+    command: --orhost or --orport 8200 --init-db -c "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci042supplychainserver"
+    logging:
+      driver: json-file
+      options:
+        max-size: "200k"
+        max-file: "10"
+
   sci033edapi:
     image: 235945186688.dkr.ecr.ap-southeast-2.amazonaws.com/edapi:latest
     restart: always

deploy/get_and_start_new_version.sh

@@ -24,7 +24,9 @@ OR_USER=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-i
 OR_PASSWORD=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .OR_PASSWORD)
 OAUTH_SUB=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .OAUTH_SUB)
 SCI033_OAUTH_SUB=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .SCI033_OAUTH_SUB)
+SCI042_OAUTH_SUB=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .SCI042_OAUTH_SUB)
 SCI041_OAUTH_SUB=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .SCI041_OAUTH_SUB)
+SCI054_OAUTH_SUB=$(aws --region ap-southeast-2 secretsmanager get-secret-value --secret-id development | jq -r .SecretString | sed 's/\\//g' | jq -r .SCI054_OAUTH_SUB)
 
 echo "POSTGRES_USER=$POSTGRES_USER
 POSTGRES_PASSWORD=$POSTGRES_PASSWORD
@@ -37,19 +39,30 @@ EDAPI_DB_CONN=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci033edapi
 DEST_HOST=sci033scs
 DEST_PORT=8000" > '/home/ec2-user/sci033.env'
 
+echo "JWK_CLIENT_IDS=$SCI042_OAUTH_SUB
+EDAPI_DB_CONN=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci042edapi
+DEST_HOST=sci042scs
+DEST_PORT=8000" > '/home/ec2-user/sci042.env'
+
 echo "JWK_CLIENT_IDS=$SCI041_OAUTH_SUB
 EDAPI_DB_CONN=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci041edapi
 DEST_HOST=sci041scs
 DEST_PORT=8000" > '/home/ec2-user/sci041.env'
 
+echo "JWK_CLIENT_IDS=$SCI054_OAUTH_SUB
+EDAPI_DB_CONN=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/sci054edapi
+DEST_HOST=sci054scs
+DEST_PORT=8000" > '/home/ec2-user/sci054.env'
+
+
 # Start the database (either an empty postgres container, or an existing database)
 docker-compose up -d db
 
 echo Waiting 10 seconds for the db to finish starting...
 sleep 10
 
 # Start the services
-docker-compose up -d web orgRegistry sci041edapi sci041scs sci033edapi sci033scs # private-ethereum-blockchain blockchain-api-server
+docker-compose up -d web orgRegistry sci033edapi sci033scs sci042edapi sci042scs sci041edapi sci041scs sci054edapi sci054scs # private-ethereum-blockchain blockchain-api-server
 
 # remove all unused docker images and exited containers
 # docker system prune -a --force

projects/mirza-common-haskell/src/Mirza/Common/Database.hs

@@ -4,40 +4,68 @@
 {-# LANGUAGE MultiParamTypeClasses #-}
 {-# LANGUAGE OverloadedStrings     #-}
 {-# LANGUAGE TypeApplications      #-}
+{-# LANGUAGE TemplateHaskell #-}
 
 module Mirza.Common.Database ( Migration
                              , runMigrationSimple
                              , dropTablesSimple
 
                              , createTrigger
+                             , createTriggerFunction
+                             , SchemaVerificationResult(..)
+                             , checkSchemaAgainstBeam
                              ) where
 
 import Mirza.Common.Types
 
 import Control.Lens (view, _1)
-import Control.Monad ( void, forM_ )
-import Data.List ( drop, zip )
+import Control.Monad (void, forM_)
+import Data.List (drop, zip)
 import Data.String (fromString)
-import Data.Text ( unpack )
+import Data.Text (unpack)
+import Database.Beam.Migrate.Simple (verifySchema, VerificationResult(..))
+import Database.Beam.Migrate.Types (CheckedDatabaseSettings)
+import Database.Beam.Postgres (Postgres, runBeamPostgres)
+import Database.Beam.Postgres.Migrate (migrationBackend)
+import Database.Beam.Schema.Tables (Database)
 import Database.PostgreSQL.Simple
+import Katip
+
+data SchemaVerificationResult = SchemaMatch | SchemaMismatch [String]
+
+checkSchemaAgainstBeam :: ( Member c '[HasLogging, HasDB]
+                          , Member err '[AsSqlError]
+                          , Database Postgres db
+                          )
+  => c -> CheckedDatabaseSettings Postgres db -> IO (Either err SchemaVerificationResult)
+checkSchemaAgainstBeam context db = runAppM context $ do
+  runDb $ do
+    conn <- view _1
+    liftIO $ runBeamPostgres conn $ verifySchema migrationBackend db >>= \case
+      VerificationSucceeded -> pure SchemaMatch
+      VerificationFailed xs -> pure $ SchemaMismatch $ fmap show xs
 
 type Migration = Connection -> IO ()
 
 runMigrationSimple :: ( Member c '[HasLogging, HasDB], Member err '[AsSqlError]) => c -> [Migration] -> IO (Either err ())
 runMigrationSimple c migrations = runAppM c $ runDb $ do
   conn <- view _1
-  liftIO $ do
-    _ <- execute_ conn "SET client_min_messages = WARNING;"
-    n <- getVersion conn
-    forM_ (drop n (zip @Int [1..] migrations)) $ \(i, m) -> do
-      m conn
-      execute conn "INSERT INTO version (number, executed) VALUES (?, now());" (Only i)
+  n <- liftIO $ getVersion conn
+
+  $(logTM) InfoS $ logStr $ mconcat [ "Database schema @ "
+                                    , show n
+                                    , "/"
+                                    , show (length migrations)
+                                    ]
+
+  forM_ (drop n (zip @Int [1..] migrations)) $ \(i, m) -> liftIO $ do
+    m conn
+    execute conn "INSERT INTO version (number, executed) VALUES (?, now());" (Only i)
 
 dropTablesSimple :: ( Member c '[HasLogging, HasDB], Member err '[AsSqlError]) => c -> IO (Either err ())
 dropTablesSimple c = runAppM c $ runDb $ do
   conn <- view _1
   liftIO $ do
-    _ <- execute_ conn "SET client_min_messages = WARNING;"
     tables <- query_ conn "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';"
     forM_ tables $ \t -> execute_ conn $ "DROP TABLE IF EXISTS " <> (fromString (unpack (fromOnly t))) <> " CASCADE;"
 
@@ -52,15 +80,17 @@ getVersion conn = do
         _ -> pure 0
     _ -> 0 <$ execute_ conn "CREATE TABLE version (number INTEGER NOT NULL PRIMARY KEY, executed timestamptz NOT NULL);"
 
-createTrigger :: Connection -> Query -> IO ()
-createTrigger conn tName = void $ execute_ conn $
+createTriggerFunction :: Connection -> IO ()
+createTriggerFunction conn = void $ execute_ conn $
       "CREATE OR REPLACE FUNCTION sync_lastmod() RETURNS trigger AS $$ \
       \BEGIN \
         \NEW.last_update := NOW() AT TIME ZONE 'UTC'; \
         \RETURN NEW; \
       \END; \
-      \$$ LANGUAGE plpgsql; \
-      \DROP TRIGGER IF EXISTS sync_lastmod ON \"" <> tName <> "\";" <>
+      \$$ LANGUAGE plpgsql;"
+
+createTrigger :: Connection -> Query -> IO ()
+createTrigger conn tName = void $ execute_ conn $
       "CREATE TRIGGER sync_lastmod \
-      \BEFORE UPDATE OR INSERT ON \"" <> tName <>
-        "\" FOR EACH ROW EXECUTE PROCEDURE sync_lastmod();"
+      \  BEFORE UPDATE OR INSERT ON \"" <> tName <> "\" \
+      \    FOR EACH ROW EXECUTE PROCEDURE sync_lastmod();"

projects/mirza-common-haskell/src/Mirza/Common/Types.hs

@@ -267,7 +267,6 @@ class AsSqlError a where
 instance AsSqlError SqlError where
   _SqlError = id
 
-
 -- Logging classes
 -- ===============
 

projects/or_scs/Mirza.cabal

@@ -40,6 +40,7 @@ library
                     , Mirza.SupplyChain.Database.Migrate
                     , Mirza.SupplyChain.Database.Schema
                     , Mirza.SupplyChain.Database.Schema.V0001
+                    , Mirza.SupplyChain.Database.Schema.SQL.V0001
                     , Mirza.SupplyChain.Handlers.Health
                     , Mirza.SupplyChain.EventUtils
                     , Mirza.SupplyChain.Handlers.Queries
@@ -68,6 +69,7 @@ library
                     , hashable
                     , hoist-error >= 0.2
                     , http-client
+                    , http-media
                     , insert-ordered-containers
                     , jose
                     , katip >= 0.5.4

projects/or_scs/src/Mirza/OrgRegistry/Database/Migrate.hs

@@ -3,14 +3,20 @@ module Mirza.OrgRegistry.Database.Migrate ( migrations
                                           , dropTablesSimple
                                           ) where
 
-import           Database.PostgreSQL.Simple
-import           Mirza.Common.Database
+import Mirza.Common.Database
+
+import Data.String ( fromString )
+import Database.PostgreSQL.Simple
+
 
 migrations :: [Migration]
 migrations = [ m_0001 ]
 
 m_0001 :: Migration
 m_0001 conn = do
+  [(Only x)] <- query_ conn "SELECT current_database();"
+  _ <- execute_ conn $ "ALTER DATABASE " <> fromString x <>" SET client_min_messages TO WARNING";
+
   _ <- execute_ conn "CREATE TABLE orgs (org_gs1_company_prefix TEXT PRIMARY KEY, org_name TEXT NOT NULL, org_url TEXT NOT NULL, last_update TIMESTAMP);"
   _ <- execute_ conn "CREATE TABLE location (location_gln TEXT PRIMARY KEY, location_org_id TEXT NOT NULL REFERENCES orgs(org_gs1_company_prefix) ON DELETE CASCADE, last_update TIMESTAMP);"
   _ <- execute_ conn "CREATE TABLE users (oauth_sub TEXT PRIMARY KEY, last_update TIMESTAMP);"
@@ -19,6 +25,7 @@ m_0001 conn = do
   _ <- execute_ conn "CREATE TABLE org_mapping (mapping_org_id TEXT NOT NULL REFERENCES orgs(org_gs1_company_prefix) ON DELETE CASCADE, mapping_user_oauth_sub TEXT NOT NULL REFERENCES users(oauth_sub) ON DELETE CASCADE, last_update TIMESTAMP, PRIMARY KEY(mapping_org_id, mapping_user_oauth_sub))"
   _ <- execute_ conn "CREATE TABLE keys (key_id UUID PRIMARY KEY, key_org TEXT NOT NULL REFERENCES orgs(org_gs1_company_prefix) ON DELETE CASCADE, jwk JSON NOT NULL, creation_time TIMESTAMP, revocation_time TIMESTAMP, revoking_user_id TEXT REFERENCES users(oauth_sub) ON DELETE CASCADE, expiration_time TIMESTAMP, last_update TIMESTAMP);"
 
+  createTriggerFunction conn
   createTrigger conn "orgs"
   createTrigger conn "location"
   createTrigger conn "users"