From d32f373e1ee95ba68141a3f3a48ab8c94ea2453a Mon Sep 17 00:00:00 2001 From: Xavier Francisco Date: Thu, 25 Apr 2019 18:23:20 +0100 Subject: [PATCH 1/7] Add config option for pool timeout (#1280) --- CHANGELOG.md | 2 ++ main/Main.hs | 4 ++-- src/PostgREST/Config.hs | 9 +++++++++ test/SpecHelper.hs | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3647e2c8c5..b3dc859bee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #690, Add `?columns` query parameter for faster bulk inserts, also ignores unspecified json keys in a payload - @steve-chavez - #1239, Add support for resource embedding on materialized views - @vitorbaptista - #1264, Add support for bulk RPC call - @steve-chavez +- #1278, Add db-pool-timeout config option - @qu4tro ### Fixed @@ -19,6 +20,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #1238, Fix PostgreSQL to OpenAPI type mappings for numeric and character types - @fpusch - #1265, Fix query generated on bulk upsert with an empty array - @qu4tro - #1273, Fix RPC ignoring unknown arguments by default - @steve-chavez +- #1257, Fix incorrect status when a PATCH request doesn't find rows to change - @qu4tro ## [5.2.0] - 2018-12-12 diff --git a/main/Main.hs b/main/Main.hs index 7d398923653..32f8d1e170e 100644 --- a/main/Main.hs +++ b/main/Main.hs @@ -4,7 +4,7 @@ module Main where import PostgREST.App (postgrest) -import PostgREST.Config (AppConfig (..), +import PostgREST.Config (AppConfig (..), configPoolTimeout', prettyVersion, readOptions) import PostgREST.DbStructure (getDbStructure, getPgVersion) import PostgREST.Error (encodeError) @@ -164,7 +164,7 @@ main = do -- -- create connection pool with the provided settings, returns either -- a 'Connection' or a 'ConnectionError'. Does not throw. - pool <- P.acquire (configPool conf, 10, pgSettings) + pool <- P.acquire (configPool conf, configPoolTimeout' conf, pgSettings) -- -- To be filled in by connectionWorker refDbStructure <- newIORef Nothing diff --git a/src/PostgREST/Config.hs b/src/PostgREST/Config.hs index 39ba4a355f3..1b18e6b768c 100644 --- a/src/PostgREST/Config.hs +++ b/src/PostgREST/Config.hs @@ -19,6 +19,7 @@ module PostgREST.Config ( prettyVersion , readOptions , corsPolicy , AppConfig (..) + , configPoolTimeout' ) where @@ -73,6 +74,7 @@ data AppConfig = AppConfig { , configJwtAudience :: Maybe StringOrURI , configPool :: Int + , configPoolTimeout :: Int , configMaxRows :: Maybe Integer , configReqCheck :: Maybe Text , configQuiet :: Bool @@ -81,6 +83,11 @@ data AppConfig = AppConfig { , configExtraSearchPath :: [Text] } +configPoolTimeout' :: (Fractional a) => AppConfig -> a +configPoolTimeout' = + fromRational . toRational . configPoolTimeout + + defaultCorsPolicy :: CorsResourcePolicy defaultCorsPolicy = CorsResourcePolicy Nothing ["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"] ["Authorization"] Nothing @@ -136,6 +143,7 @@ readOptions = do <*> (fromMaybe False . join . fmap coerceBool <$> C.key "secret-is-base64") <*> parseJwtAudience "jwt-aud" <*> (fromMaybe 10 . join . fmap coerceInt <$> C.key "db-pool") + <*> (fromMaybe 10 . join . fmap coerceInt <$> C.key "db-pool-timeout") <*> (join . fmap coerceInt <$> C.key "max-rows") <*> (mfilter (/= "") <$> C.key "pre-request") <*> pure False @@ -208,6 +216,7 @@ readOptions = do |db-schema = "public" # this schema gets added to the search_path of every request |db-anon-role = "postgres" |db-pool = 10 + |db-pool-timeout = 10 | |server-host = "127.0.0.1" |server-port = 3000 diff --git a/test/SpecHelper.hs b/test/SpecHelper.hs index 403b5104d81..5a2c3b9df79 100644 --- a/test/SpecHelper.hs +++ b/test/SpecHelper.hs @@ -70,7 +70,7 @@ _baseCfg = -- Connection Settings -- Jwt settings (Just $ encodeUtf8 "reallyreallyreallyreallyverysafe") False Nothing -- Connection Modifiers - 10 Nothing (Just "test.switch_role") + 10 10 Nothing (Just "test.switch_role") -- Debug Settings True [ ("app.settings.app_host", "localhost") From bcc317db81243dcd5ce488f445e3bb5e9a0fa62e Mon Sep 17 00:00:00 2001 From: David Ernst Date: Fri, 26 Apr 2019 17:46:20 -0700 Subject: [PATCH 2/7] Fix reference to deprecated SIGUP call (#1284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This updates the instructions about sending a `SIGUP` call (deprecated) to `SIGUSR1`, and links to relevant docs: http://postgrest.org/en/v5.2/admin.html#schema-reloading Per the docs: > As of PostgREST v5.1 reloading with SIGHUP is deprecated, it’s still supported but will be removed in v6.0. SIGUSR1 should be used instead. See [5.1.0 CHANGELOG](https://github.com/PostgREST/postgrest/blob/d32f373e1ee95ba68141a3f3a48ab8c94ea2453a/CHANGELOG.md#deprecated) or https://github.com/PostgREST/postgrest/issues/724 --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 153cb140c57..8bf344c3ed7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -32,7 +32,7 @@ your contributions. then [find your logs](http://blog.endpoint.com/2014/11/dear-postgresql-where-are-my-logs.html). * If your database schema has changed while the PostgREST server is running, - send the server a `SIGHUP` signal or restart it to ensure the schema cache + [send the server a `SIGUSR1` signal](http://postgrest.org/en/v5.2/admin.html#schema-reloading) or restart it to ensure the schema cache is not stale. This sometimes fixes apparent bugs. ## Code From 37de12d3763c9d906dc4cd5a012db6030adf660a Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Mon, 29 Apr 2019 14:12:30 -0500 Subject: [PATCH 3/7] Remove support for schema reloading with SIGHUP SIGUSR1 should be used instead --- CHANGELOG.md | 4 ++++ main/Main.hs | 17 ++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3dc859bee2..d42bf9f132b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #1273, Fix RPC ignoring unknown arguments by default - @steve-chavez - #1257, Fix incorrect status when a PATCH request doesn't find rows to change - @qu4tro +### Removed + +- #1288, Removed support for schema reloading with SIGHUP, SIGUSR1 should be used instead - @steve-chavez + ## [5.2.0] - 2018-12-12 ### Added diff --git a/main/Main.hs b/main/Main.hs index 32f8d1e170e..5d7f51afd5a 100644 --- a/main/Main.hs +++ b/main/Main.hs @@ -198,15 +198,14 @@ main = do throwTo mainTid UserInterrupt ) Nothing - forM_ [sigHUP, sigUSR1] $ \sig -> - void $ installHandler sig ( - Catch $ connectionWorker - mainTid - pool - (configSchema conf) - refDbStructure - refIsWorkerOn - ) Nothing + void $ installHandler sigUSR1 ( + Catch $ connectionWorker + mainTid + pool + (configSchema conf) + refDbStructure + refIsWorkerOn + ) Nothing #endif From 181b608c04a532149359e1f302e195e7a43fe4d8 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Mon, 29 Apr 2019 14:14:42 -0500 Subject: [PATCH 4/7] Change server-host default of 127.0.0.1 to !4 This way the server works in LAN settings by default. --- CHANGELOG.md | 4 ++++ src/PostgREST/Config.hs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d42bf9f132b..48c2be725d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #1273, Fix RPC ignoring unknown arguments by default - @steve-chavez - #1257, Fix incorrect status when a PATCH request doesn't find rows to change - @qu4tro +### Changed + +- #1288, Change server-host default of 127.0.0.1 to !4 + ### Removed - #1288, Removed support for schema reloading with SIGHUP, SIGUSR1 should be used instead - @steve-chavez diff --git a/src/PostgREST/Config.hs b/src/PostgREST/Config.hs index 1b18e6b768c..c187d911226 100644 --- a/src/PostgREST/Config.hs +++ b/src/PostgREST/Config.hs @@ -137,7 +137,7 @@ readOptions = do <*> C.key "db-anon-role" <*> (mfilter (/= "") <$> C.key "server-proxy-uri") <*> C.key "db-schema" - <*> (fromMaybe "127.0.0.1" . mfilter (/= "") <$> C.key "server-host") + <*> (fromMaybe "!4" . mfilter (/= "") <$> C.key "server-host") <*> (fromMaybe 3000 . join . fmap coerceInt <$> C.key "server-port") <*> (fmap encodeUtf8 . mfilter (/= "") <$> C.key "jwt-secret") <*> (fromMaybe False . join . fmap coerceBool <$> C.key "secret-is-base64") @@ -218,7 +218,7 @@ readOptions = do |db-pool = 10 |db-pool-timeout = 10 | - |server-host = "127.0.0.1" + |server-host = "!4" |server-port = 3000 | |## base url for swagger output From 3c7738a8c7b0f14ebac25fbed4f8d3f4f5cb89fb Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Tue, 30 Apr 2019 16:31:13 -0500 Subject: [PATCH 5/7] Deprecate '.' for disambiguating resource embedding '+' should be used instead. --- CHANGELOG.md | 4 ++++ src/PostgREST/ApiRequest.hs | 11 ++++++----- src/PostgREST/Parsers.hs | 5 ++++- stack.yaml | 5 +++++ test/Feature/QuerySpec.hs | 27 ++++++++++++++++++--------- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c2be725d1..75e4040af3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #1288, Change server-host default of 127.0.0.1 to !4 +### Deprecated + +- #1288, Deprecate `.` symbol for disambiguating resource embedding(added in #918). '+' should be used instead. Though '+' is url safe, certain clients might need to encode it to '%2B'. + ### Removed - #1288, Removed support for schema reloading with SIGHUP, SIGUSR1 should be used instead - @steve-chavez diff --git a/src/PostgREST/ApiRequest.hs b/src/PostgREST/ApiRequest.hs index 7c2a1ff82bb..5a862c6f29f 100644 --- a/src/PostgREST/ApiRequest.hs +++ b/src/PostgREST/ApiRequest.hs @@ -29,7 +29,7 @@ import qualified Data.Text as T import qualified Data.Vector as V import Network.HTTP.Base (urlEncodeVars) import Network.HTTP.Types.Header (hAuthorization, hCookie) -import Network.HTTP.Types.URI (parseSimpleQuery) +import Network.HTTP.Types.URI (parseSimpleQuery, parseQueryReplacePlus) import Network.Wai (Request (..)) import Network.Wai.Parse (parseHttpAccept) import PostgREST.RangeQuery (NonnegRange, rangeRequested, restrictRange, rangeGeq, allRange, rangeLimit, rangeOffset) @@ -127,14 +127,15 @@ userApiRequest schema req reqBody , iOrder = [(toS k, toS $ fromJust v) | (k,v) <- qParams, isJust v, endingIn ["order"] k ] , iCanonicalQS = toS $ urlEncodeVars . L.sortBy (comparing fst) - . map (join (***) toS) - . parseSimpleQuery - $ rawQueryString req + . map (join (***) toS . second (fromMaybe BS.empty)) + $ queryStringWPlus , iJWT = tokenStr , iHeaders = [ (toS $ CI.foldedCase k, toS v) | (k,v) <- hdrs, k /= hAuthorization, k /= hCookie] , iCookies = maybe [] parseCookiesText $ lookupHeader "Cookie" } where + -- queryString with '+' not converted to ' ' + queryStringWPlus = parseQueryReplacePlus False $ rawQueryString req -- rpcQParams = Rpc query params e.g. /rpc/name?param1=val1, similar to filter but with no operator(eq, lt..) (filters, rpcQParams) = case action of @@ -201,7 +202,7 @@ userApiRequest schema req reqBody path = pathInfo req method = requestMethod req hdrs = requestHeaders req - qParams = [(toS k, v)|(k,v) <- queryString req] + qParams = [(toS k, v)|(k,v) <- queryStringWPlus] lookupHeader = flip lookup hdrs hasPrefer :: Text -> Bool hasPrefer val = any (\(h,v) -> h == "Prefer" && val `elem` split v) hdrs diff --git a/src/PostgREST/Parsers.hs b/src/PostgREST/Parsers.hs index 1ccc01bd8e7..fee6b368a18 100644 --- a/src/PostgREST/Parsers.hs +++ b/src/PostgREST/Parsers.hs @@ -124,7 +124,10 @@ pRelationSelect :: Parser SelectItem pRelationSelect = lexeme $ try ( do alias <- optionMaybe ( try(pFieldName <* aliasSeparator) ) fld <- pField - relationDetail <- optionMaybe ( try( char '.' *> pFieldName ) ) + relationDetail <- optionMaybe ( + try ( char '+' *> pFieldName ) <|> + try ( char '.' *> pFieldName ) -- TODO deprecated, remove in next major version + ) return (fld, Nothing, alias, relationDetail) ) diff --git a/stack.yaml b/stack.yaml index dbfc05744c7..0f54fbead76 100644 --- a/stack.yaml +++ b/stack.yaml @@ -12,7 +12,12 @@ extra-deps: - text-builder-0.5.1.1 - jose-0.7.0.0 - postgresql-libpq-0.9.4.1 + - http-types-0.12.3 + - wai-middleware-static-0.8.2 ghc-options: postgrest: -O2 -Werror -Wall -fwarn-identities -fno-warn-redundant-constraints nix: packages: [postgresql, zlib] +# only added because of hjsonschema conflict with http-types +# once hjsonschema upper bounding on http-types is solved it can be removed +allow-newer: true diff --git a/test/Feature/QuerySpec.hs b/test/Feature/QuerySpec.hs index 3b65ff0c433..2b2e87e079c 100644 --- a/test/Feature/QuerySpec.hs +++ b/test/Feature/QuerySpec.hs @@ -443,12 +443,12 @@ spec = do describe "path fixed" $ do it "works when requesting children 2 levels" $ - get "/clients?id=eq.1&select=id,projects:projects.client_id(id,tasks(id))" `shouldRespondWith` + get "/clients?id=eq.1&select=id,projects:projects%2Bclient_id(id,tasks(id))" `shouldRespondWith` [json|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":3},{"id":4}]}]}]|] { matchHeaders = [matchContentTypeJson] } it "works with parent relation" $ - get "/message?select=id,body,sender:person.sender(name),recipient:person.recipient(name)&id=lt.4" `shouldRespondWith` + get "/message?select=id,body,sender:person%2Bsender(name),recipient:person%2Brecipient(name)&id=lt.4" `shouldRespondWith` [json| [{"id":1,"body":"Hello Jane","sender":{"name":"John"},"recipient":{"name":"Jane"}}, {"id":2,"body":"Hi John","sender":{"name":"Jane"},"recipient":{"name":"John"}}, @@ -456,7 +456,7 @@ spec = do { matchHeaders = [matchContentTypeJson] } it "works with a parent view relation" $ - get "/message?select=id,body,sender:person_detail.sender(name,sent),recipient:person_detail.recipient(name,received)&id=lt.4" `shouldRespondWith` + get "/message?select=id,body,sender:person_detail%2Bsender(name,sent),recipient:person_detail%2Brecipient(name,received)&id=lt.4" `shouldRespondWith` [json| [{"id":1,"body":"Hello Jane","sender":{"name":"John","sent":2},"recipient":{"name":"Jane","received":2}}, {"id":2,"body":"Hi John","sender":{"name":"Jane","sent":1},"recipient":{"name":"John","received":1}}, @@ -464,10 +464,19 @@ spec = do { matchHeaders = [matchContentTypeJson] } it "works with many<->many relation" $ - get "/tasks?select=id,users:users.users_tasks(id)" `shouldRespondWith` + get "/tasks?select=id,users:users%2Busers_tasks(id)" `shouldRespondWith` [json|[{"id":1,"users":[{"id":1},{"id":3}]},{"id":2,"users":[{"id":1}]},{"id":3,"users":[{"id":1}]},{"id":4,"users":[{"id":1}]},{"id":5,"users":[{"id":2},{"id":3}]},{"id":6,"users":[{"id":2}]},{"id":7,"users":[{"id":2}]},{"id":8,"users":[]}]|] { matchHeaders = [matchContentTypeJson] } + describe "old dot '.' symbol, deprecated" $ + it "still works" $ do + get "/clients?id=eq.1&select=id,projects:projects.client_id(id,tasks(id))" `shouldRespondWith` + [json|[{"id":1,"projects":[{"id":1,"tasks":[{"id":1},{"id":2}]},{"id":2,"tasks":[{"id":3},{"id":4}]}]}]|] + { matchHeaders = [matchContentTypeJson] } + get "/tasks?select=id,users:users.users_tasks(id)" `shouldRespondWith` + [json|[{"id":1,"users":[{"id":1},{"id":3}]},{"id":2,"users":[{"id":1}]},{"id":3,"users":[{"id":1}]},{"id":4,"users":[{"id":1}]},{"id":5,"users":[{"id":2},{"id":3}]},{"id":6,"users":[{"id":2}]},{"id":7,"users":[{"id":2}]},{"id":8,"users":[]}]|] + { matchHeaders = [matchContentTypeJson] } + describe "aliased embeds" $ do it "works with child relation" $ get "/space?select=id,zones:zone(id,name),stores:zone(id,name)&zones.zone_type_id=eq.2&stores.zone_type_id=eq.3" `shouldRespondWith` @@ -536,7 +545,7 @@ spec = do { matchHeaders = [matchContentTypeJson] } it "embeds childs recursively" $ - get "/family_tree?id=eq.1&select=id,name, childs:family_tree.parent(id,name,childs:family_tree.parent(id,name))" `shouldRespondWith` + get "/family_tree?id=eq.1&select=id,name, childs:family_tree%2Bparent(id,name,childs:family_tree%2Bparent(id,name))" `shouldRespondWith` [json|[{ "id": "1", "name": "Parental Unit", "childs": [ { "id": "2", "name": "Kid One", "childs": [ { "id": "4", "name": "Grandkid One" } ] }, @@ -545,7 +554,7 @@ spec = do }]|] { matchHeaders = [matchContentTypeJson] } it "embeds parent and then embeds childs" $ - get "/family_tree?id=eq.2&select=id,name,parent(id,name,childs:family_tree.parent(id,name))" `shouldRespondWith` + get "/family_tree?id=eq.2&select=id,name,parent(id,name,childs:family_tree%2Bparent(id,name))" `shouldRespondWith` [json|[{ "id": "2", "name": "Kid One", "parent": { "id": "1", "name": "Parental Unit", "childs": [ { "id": "2", "name": "Kid One" }, { "id": "3", "name": "Kid Two"} ] @@ -568,7 +577,7 @@ spec = do }]|] { matchHeaders = [matchContentTypeJson] } it "embeds childs" $ do - get "/organizations?select=id,name,refereeds:organizations.referee(id,name)&id=eq.1" `shouldRespondWith` + get "/organizations?select=id,name,refereeds:organizations%2Breferee(id,name)&id=eq.1" `shouldRespondWith` [json|[{ "id": 1, "name": "Referee Org", "refereeds": [ @@ -582,7 +591,7 @@ spec = do } ] }]|] { matchHeaders = [matchContentTypeJson] } - get "/organizations?select=id,name,auditees:organizations.auditor(id,name)&id=eq.2" `shouldRespondWith` + get "/organizations?select=id,name,auditees:organizations%2Bauditor(id,name)&id=eq.2" `shouldRespondWith` [json|[{ "id": 2, "name": "Auditor Org", "auditees": [ @@ -616,7 +625,7 @@ spec = do "manager":{"name":"Referee Manager"}}} }]|] { matchHeaders = [matchContentTypeJson] } - get "/organizations?select=name,manager(name),auditees:organizations.auditor(name,manager(name),refereeds:organizations.referee(name,manager(name)))&id=eq.2" `shouldRespondWith` + get "/organizations?select=name,manager(name),auditees:organizations%2Bauditor(name,manager(name),refereeds:organizations%2Breferee(name,manager(name)))&id=eq.2" `shouldRespondWith` [json|[{ "name":"Auditor Org", "manager":{"name":"Auditor Manager"}, From 3078a1114454ed289d8ab9ffeb72a03c44b935a7 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Sat, 18 May 2019 18:05:56 -0500 Subject: [PATCH 6/7] Remove circleci extra cabal checks Currently these extra checks are giving an out of memory error on CircleCI: Hint: Max memory usage of this container is 4231065600 --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5eef71ac74b..99935ed2396 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,15 +91,6 @@ jobs: - run: name: run linter command: git ls-files | grep '\.l\?hs$' | xargs stack exec -- hlint -X QuasiQuotes -X NoPatternSynonyms "$@" - - run: - name: extra checks - command: | - stack exec -- cabal update - stack exec --no-ghc-package-path -- cabal install --only-d --dry-run - stack exec -- packdeps *.cabal || true - stack exec -- cabal check - stack haddock --no-haddock-deps - stack sdist - save_cache: paths: - "~/.stack" From 8d8374cef0b438c52789a1822df02a290ee5f3ea Mon Sep 17 00:00:00 2001 From: Russell Davies Date: Mon, 13 May 2019 11:54:45 +0100 Subject: [PATCH 7/7] Update Dockerfile base image to debian:stretch-slim --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7162c6bb08c..2d99776ea28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:jessie +FROM debian:stretch-slim ARG POSTGREST_VERSION