From 125ea8f6d9ecfe40ed4a1c5a9291ae23bb695972 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Mon, 18 Jan 2021 18:47:37 -0500 Subject: [PATCH] feat: allow reloading config with NOTIFY Enables reloading the config by doing: NOTIFY pgrst, 'reload config' Adds an alias for reloading the schema cache: NOTIFY pgrst, 'reload schema' --- main/Main.hs | 50 +++++++-------- test/fixtures/roles.sql | 2 +- test/fixtures/schema.sql | 26 +++++++- .../expected/no-defaults-with-db.config | 2 +- test/io-tests/test_io.py | 63 ++++++++++++++++++- 5 files changed, 110 insertions(+), 33 deletions(-) diff --git a/main/Main.hs b/main/Main.hs index 761e1bbc4a..44f91a9f08 100644 --- a/main/Main.hs +++ b/main/Main.hs @@ -1,4 +1,6 @@ -{-# LANGUAGE CPP #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE NamedFieldPuns #-} module Main (main) where @@ -13,10 +15,6 @@ import qualified Hasql.Transaction.Sessions as HT import Control.AutoUpdate (defaultUpdateSettings, mkAutoUpdate, updateAction) -import Control.Debounce (debounceAction, debounceEdge, - debounceFreq, - defaultDebounceSettings, mkDebounce, - trailingEdge) import Control.Retry (RetryStatus, capDelay, exponentialBackoff, retrying, rsPreviousDelay) @@ -64,11 +62,11 @@ main = do -- read PGRST_ env variables env <- readEnvironment - -- read path from commad line - opts <- readCLIShowHelp env + -- read command/path from commad line + CLI{cliCommand, cliPath} <- readCLIShowHelp env -- build the 'AppConfig' from the config file path - conf <- readValidateConfig mempty env $ cliPath opts + conf <- readValidateConfig mempty env cliPath -- These are config values that can't be reloaded at runtime. Reloading some of them would imply restarting the web server. let @@ -105,11 +103,12 @@ main = do -- Config that can change at runtime refConf <- newIORef conf + let configRereader = reReadConfig pool gucConfigEnabled env cliPath refConf + -- re-read and override the config if db-load-guc-config is true - when gucConfigEnabled $ - reReadConfig pool gucConfigEnabled env (cliPath opts) refConf + when gucConfigEnabled configRereader - case cliCommand opts of + case cliCommand of CmdDumpConfig -> do dumpedConfig <- dumpAppConfig <$> readIORef refConf @@ -149,13 +148,13 @@ main = do -- Re-read the config on SIGUSR2 void $ installHandler sigUSR2 ( - Catch $ reReadConfig pool gucConfigEnabled env (cliPath opts) refConf >> putStrLn ("Config reloaded" :: Text) + Catch $ configRereader >> putStrLn ("Config reloaded" :: Text) ) Nothing #endif - -- reload schema cache on NOTIFY + -- reload schema cache + config on NOTIFY when dbChannelEnabled $ - listener dbUri dbChannel pool refConf refDbStructure mvarConnectionStatus connWorker + listener dbUri dbChannel pool refConf refDbStructure mvarConnectionStatus connWorker configRereader -- ask for the OS time at most once per second getTime <- mkAutoUpdate defaultUpdateSettings {updateAction = getCurrentTime} @@ -310,32 +309,27 @@ loadSchemaCache pool actualPgVersion refConf refDbStructure = do When a NOTIFY - with an empty payload - is done, it refills the schema cache. It uses the connectionWorker in case the LISTEN connection dies. -} -listener :: ByteString -> Text -> P.Pool -> IORef AppConfig -> IORef (Maybe DbStructure) -> MVar ConnectionStatus -> IO () -> IO () -listener dbUri dbChannel pool refConf refDbStructure mvarConnectionStatus connWorker = start +listener :: ByteString -> Text -> P.Pool -> IORef AppConfig -> IORef (Maybe DbStructure) -> MVar ConnectionStatus -> IO () -> IO () -> IO () +listener dbUri dbChannel pool refConf refDbStructure mvarConnectionStatus connWorker configRereader = start where start = do connStatus <- takeMVar mvarConnectionStatus -- takeMVar makes the thread wait if the MVar is empty(until there's a connection). case connStatus of Connected actualPgVersion -> void $ forkFinally (do -- forkFinally allows to detect if the thread dies dbOrError <- C.acquire dbUri - -- Debounce in case too many NOTIFYs arrive. Could happen on a migration(assuming a pg EVENT TRIGGER is set up). - -- This might not be needed according to pg docs https://www.postgresql.org/docs/12/sql-notify.html: - -- "If the same channel name is signaled multiple times from the same transaction with identical payload strings, the database server can decide to deliver a single notification only." - -- But we do it to be extra safe. - scFiller <- mkDebounce (defaultDebounceSettings { - -- It's not necessary to check the loadSchemaCache success here. If the connection drops, the thread will die and proceed to recover below. - debounceAction = void $ loadSchemaCache pool actualPgVersion refConf refDbStructure, - debounceEdge = trailingEdge, -- wait until the function hasn’t been called in _1s - debounceFreq = _1s }) case dbOrError of Right db -> do putStrLn $ "Listening for notifications on the " <> dbChannel <> " channel" let channelToListen = N.toPgIdentifier dbChannel + cfLoader = configRereader >> putStrLn ("Config reloaded" :: Text) + scLoader = void $ loadSchemaCache pool actualPgVersion refConf refDbStructure -- It's not necessary to check the loadSchemaCache success here. If the connection drops, the thread will die and proceed to recover below. N.listen db channelToListen N.waitForNotifications (\_ msg -> - if BS.null msg - then scFiller -- reload the schema cache - else pure ()) db -- Do nothing if anything else than an empty message is sent + if | BS.null msg -> scLoader -- reload the schema cache + | msg == "reload schema" -> scLoader -- reload the schema cache + | msg == "reload config" -> cfLoader -- reload the config + | otherwise -> pure () -- Do nothing if anything else than an empty message is sent + ) db _ -> die errorMessage) (\_ -> do -- if the thread dies, we try to recover putStrLn retryMessage diff --git a/test/fixtures/roles.sql b/test/fixtures/roles.sql index 53a9eafb49..286065b7a5 100644 --- a/test/fixtures/roles.sql +++ b/test/fixtures/roles.sql @@ -17,7 +17,7 @@ ALTER ROLE postgrest_test_authenticator SET pgrst."db-tx-end" = 'commit-allow-ov ALTER ROLE postgrest_test_authenticator SET pgrst."db-schemas" = 'test, tenant1, tenant2'; ALTER ROLE postgrest_test_authenticator SET pgrst."db-root-spec" = 'root'; ALTER ROLE postgrest_test_authenticator SET pgrst."db-prepared-statements" = 'false'; -ALTER ROLE postgrest_test_authenticator SET pgrst."db-pre-request" = 'custom_headers'; +ALTER ROLE postgrest_test_authenticator SET pgrst."db-pre-request" = 'test.custom_headers'; ALTER ROLE postgrest_test_authenticator SET pgrst."db-max-rows" = '1000'; ALTER ROLE postgrest_test_authenticator SET pgrst."db-extra-search-path" = 'public, extensions'; diff --git a/test/fixtures/schema.sql b/test/fixtures/schema.sql index 7248540b98..09bf974ede 100644 --- a/test/fixtures/schema.sql +++ b/test/fixtures/schema.sql @@ -1925,9 +1925,33 @@ $$ language sql; create view prepared_statements as select * from pg_catalog.pg_prepared_statements; -create or replace function change_max_rows_config(val int) returns void as $_$ +create or replace function change_max_rows_config(val int, notify bool default false) returns void as $_$ begin execute format($$ alter role postgrest_test_authenticator set pgrst."db-max-rows" = %L; $$, val); + if notify then + perform pg_notify('pgrst', 'reload config'); + end if; +end $_$ volatile security definer language plpgsql ; + +create or replace function reset_max_rows_config() returns void as $_$ +begin + alter role postgrest_test_authenticator set pgrst."db-max-rows" = '1000'; +end $_$ volatile security definer language plpgsql ; + +create or replace function change_db_schema_and_full_reload(schemas text) returns void as $_$ +begin + execute format($$ + alter role postgrest_test_authenticator set pgrst."db-schemas" = %L; + $$, schemas); + perform pg_notify('pgrst', 'reload config'); + perform pg_notify('pgrst', 'reload schema'); +end $_$ volatile security definer language plpgsql ; + +create or replace function v1.reset_db_schema_config() returns void as $_$ +begin + alter role postgrest_test_authenticator set pgrst."db-schemas" = 'test'; + perform pg_notify('pgrst', 'reload config'); + perform pg_notify('pgrst', 'reload schema'); end $_$ volatile security definer language plpgsql ; diff --git a/test/io-tests/configs/expected/no-defaults-with-db.config b/test/io-tests/configs/expected/no-defaults-with-db.config index 67d268bd93..670178ff31 100644 --- a/test/io-tests/configs/expected/no-defaults-with-db.config +++ b/test/io-tests/configs/expected/no-defaults-with-db.config @@ -5,7 +5,7 @@ db-extra-search-path = "public,extensions" db-max-rows = 1000 db-pool = 1 db-pool-timeout = 100 -db-pre-request = "custom_headers" +db-pre-request = "test.custom_headers" db-prepared-statements = false db-root-spec = "root" db-schemas = "test,tenant1,tenant2" diff --git a/test/io-tests/test_io.py b/test/io-tests/test_io.py index 2976db5c1f..4d54e3ecdd 100644 --- a/test/io-tests/test_io.py +++ b/test/io-tests/test_io.py @@ -87,7 +87,7 @@ def defaultenv(): "PGRST_DB_URI": os.environ["PGRST_DB_URI"], "PGRST_DB_SCHEMAS": os.environ["PGRST_DB_SCHEMAS"], "PGRST_DB_ANON_ROLE": os.environ["PGRST_DB_ANON_ROLE"], - "PGRST_DB_LOAD_GUC_CONFIG": "false" + "PGRST_DB_LOAD_GUC_CONFIG": "false", } @@ -187,7 +187,6 @@ def wait_until_ready(url): for _ in range(10): try: response = session.get(url, timeout=1) - if response.status_code == 200: return except (requests.ConnectionError, requests.ReadTimeout): @@ -508,6 +507,34 @@ def test_db_schema_reload(tmp_path, defaultenv): assert response.status_code == 200 +def test_db_schema_notify_reload(defaultenv): + "DB schema and config should be reloaded when PostgREST is sent a NOTIFY" + + env = { + **defaultenv, + "PGRST_DB_LOAD_GUC_CONFIG": "true", + "PGRST_DB_CHANNEL_ENABLED": "true", + "PGRST_DB_SCHEMAS": "test", + } + + with run(env=env) as postgrest: + response = postgrest.session.get("/parents") + assert response.status_code == 404 + + # change db-schemas config on the db and reload config and cache with notify + postgrest.session.post( + "/rpc/change_db_schema_and_full_reload", data={"schemas": "v1"} + ) + + time.sleep(0.5) + + response = postgrest.session.get("/parents?select=*,children(*)") + assert response.status_code == 200 + + # reset db-schemas config on the db + postgrest.session.post("/rpc/reset_db_schema_config") + + def test_max_rows_reload(defaultenv): "max-rows should be reloaded from role settings when PostgREST receives a SIGUSR2." config = CONFIGSDIR / "sigusr2-settings.config" @@ -530,4 +557,36 @@ def test_max_rows_reload(defaultenv): time.sleep(0.1) response = postgrest.session.head("/projects") + assert response.headers["Content-Range"] == "0-0/*" + + # reset max-rows config on the db + postgrest.session.post("/rpc/reset_max_rows_config") + + +def test_max_rows_notify_reload(defaultenv): + "max-rows should be reloaded from role settings when PostgREST receives a NOTIFY" + + env = { + **defaultenv, + "PGRST_DB_LOAD_GUC_CONFIG": "true", + "PGRST_DB_CHANNEL_ENABLED": "true", + } + + with run(env=env) as postgrest: + response = postgrest.session.head("/projects") + assert response.headers["Content-Range"] == "0-4/*" + + # change max-rows config on the db and reload with notify + postgrest.session.post( + "/rpc/change_max_rows_config", data={"val": 1, "notify": True} + ) + + time.sleep(0.1) + + response = postgrest.session.head("/projects") + + assert response.headers["Content-Range"] == "0-0/*" + + # reset max-rows config on the db + postgrest.session.post("/rpc/reset_max_rows_config")