From c33ca4e60c7c1528bd14a682a77812ea2feecb50 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 28 Mar 2024 10:38:00 -0500 Subject: [PATCH 1/5] refactor: dry timings calculation for openapi --- src/PostgREST/ApiRequest.hs | 6 ++--- src/PostgREST/App.hs | 14 ++++-------- src/PostgREST/Plan.hs | 24 ++++++++++++++------ src/PostgREST/Query.hs | 45 ++++++++++++++++++++----------------- src/PostgREST/Response.hs | 33 +++++++++++++-------------- 5 files changed, 64 insertions(+), 58 deletions(-) diff --git a/src/PostgREST/ApiRequest.hs b/src/PostgREST/ApiRequest.hs index a58d7b1e06b..b89b70da6f6 100644 --- a/src/PostgREST/ApiRequest.hs +++ b/src/PostgREST/ApiRequest.hs @@ -94,10 +94,10 @@ data DbAction = ActRelationRead {dbActQi :: QualifiedIdentifier, actHeadersOnly :: Bool} | ActRelationMut {dbActQi :: QualifiedIdentifier, actMutation :: Mutation} | ActRoutine {dbActQi :: QualifiedIdentifier, actInvMethod :: InvokeMethod} + | ActSchemaRead Schema Bool data Action = ActDb DbAction - | ActSchemaRead Schema Bool | ActRelationInfo QualifiedIdentifier | ActRoutineInfo QualifiedIdentifier | ActSchemaInfo @@ -189,8 +189,8 @@ getAction resource schema method = (ResourceRelation rel, "DELETE") -> Right . ActDb $ ActRelationMut (qi rel) MutationDelete (ResourceRelation rel, "OPTIONS") -> Right $ ActRelationInfo (qi rel) - (ResourceSchema, "HEAD") -> Right $ ActSchemaRead schema True - (ResourceSchema, "GET") -> Right $ ActSchemaRead schema False + (ResourceSchema, "HEAD") -> Right . ActDb $ ActSchemaRead schema True + (ResourceSchema, "GET") -> Right . ActDb $ ActSchemaRead schema False (ResourceSchema, "OPTIONS") -> Right ActSchemaInfo _ -> Left $ UnsupportedMethod method diff --git a/src/PostgREST/App.hs b/src/PostgREST/App.hs index ccc16bdd3ba..ebd288fef0c 100644 --- a/src/PostgREST/App.hs +++ b/src/PostgREST/App.hs @@ -172,14 +172,8 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A case iAction of ActDb dbAct -> do (planTime', plan) <- withTiming $ liftEither $ Plan.actionPlan dbAct conf apiReq sCache - (txTime', resultSet) <- withTiming $ runQuery (planIsoLvl plan) (planFunSettings plan) (Plan.pTxMode plan) $ Query.actionQuery plan conf apiReq pgVer - (respTime', pgrst) <- withTiming $ liftEither $ Response.actionResponse plan (dbActQi dbAct) apiReq resultSet - return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst - - ActSchemaRead tSchema headersOnly -> do - (planTime', iPlan) <- withTiming $ liftEither $ Plan.inspectPlan apiReq headersOnly tSchema - (txTime', oaiResult) <- withTiming $ runQuery roleIsoLvl mempty (Plan.ipTxmode iPlan) $ Query.openApiQuery iPlan conf sCache pgVer - (respTime', pgrst) <- withTiming $ liftEither $ Response.openApiResponse iPlan (T.decodeUtf8 prettyVersion, docsVersion) oaiResult conf sCache iSchema iNegotiatedByProfile + (txTime', queryResult) <- withTiming $ runQuery (planIsoLvl plan) (planFunSettings plan) (Plan.actionPlanTxMode plan) $ Query.actionQuery plan conf apiReq pgVer sCache + (respTime', pgrst) <- withTiming $ liftEither $ Response.actionResponse queryResult (dbActQi dbAct) apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache iSchema iNegotiatedByProfile return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst ActRelationInfo identifier -> do @@ -204,10 +198,10 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A Query.runPreReq conf query - planIsoLvl (Plan.CallReadPlan{crProc}) = fromMaybe roleIsoLvl $ pdIsoLvl crProc + planIsoLvl (Plan.Db Plan.CallReadPlan{crProc}) = fromMaybe roleIsoLvl $ pdIsoLvl crProc planIsoLvl _ = roleIsoLvl - planFunSettings (Plan.CallReadPlan{crProc}) = pdFuncSettings crProc + planFunSettings (Plan.Db Plan.CallReadPlan{crProc}) = pdFuncSettings crProc planFunSettings _ = mempty pgrstResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response diff --git a/src/PostgREST/Plan.hs b/src/PostgREST/Plan.hs index 43b67a7ae05..25d3013d0fa 100644 --- a/src/PostgREST/Plan.hs +++ b/src/PostgREST/Plan.hs @@ -18,9 +18,11 @@ resource. module PostgREST.Plan ( actionPlan , ActionPlan(..) + , DbActionPlan(..) , InspectPlan(..) , inspectPlan , callReadPlan + , actionPlanTxMode ) where import qualified Data.ByteString.Lazy as LBS @@ -90,7 +92,7 @@ import Protolude hiding (from) -- Setup for doctests -- >>> import Data.Ranged.Ranges (fullRange) -data ActionPlan +data DbActionPlan = WrappedReadPlan { wrReadPlan :: ReadPlanTree , pTxMode :: SQL.Mode @@ -123,23 +125,31 @@ data InspectPlan = InspectPlan { , ipSchema :: Schema } +data ActionPlan = Db DbActionPlan | MaybeDb InspectPlan + +actionPlanTxMode :: ActionPlan -> SQL.Mode +actionPlanTxMode (Db x) = pTxMode x +actionPlanTxMode (MaybeDb x) = ipTxmode x + actionPlan :: DbAction -> AppConfig -> ApiRequest -> SchemaCache -> Either Error ActionPlan actionPlan dbAct conf apiReq sCache = case dbAct of ActRelationRead identifier headersOnly -> - wrappedReadPlan identifier conf sCache apiReq headersOnly + Db <$> wrappedReadPlan identifier conf sCache apiReq headersOnly ActRelationMut identifier mut -> - mutateReadPlan mut apiReq identifier conf sCache + Db <$> mutateReadPlan mut apiReq identifier conf sCache ActRoutine identifier invMethod -> - callReadPlan identifier conf sCache apiReq invMethod + Db <$> callReadPlan identifier conf sCache apiReq invMethod + ActSchemaRead tSchema headersOnly -> + MaybeDb <$> inspectPlan apiReq headersOnly tSchema -wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error ActionPlan +wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> Bool -> Either Error DbActionPlan wrappedReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} headersOnly = do rPlan <- readPlan identifier conf sCache apiRequest (handler, mediaType) <- mapLeft ApiRequestError $ negotiateContent conf apiRequest identifier iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan) if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right () return $ WrappedReadPlan rPlan SQL.Read handler mediaType headersOnly -mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error ActionPlan +mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error DbActionPlan mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do rPlan <- readPlan identifier conf sCache apiRequest mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan @@ -147,7 +157,7 @@ mutateReadPlan mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} (handler, mediaType) <- mapLeft ApiRequestError $ negotiateContent conf apiRequest identifier iAcceptMediaType (dbMediaHandlers sCache) (hasDefaultSelect rPlan) return $ MutateReadPlan rPlan mPlan SQL.Write handler mediaType mutation -callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error ActionPlan +callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error DbActionPlan callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} invMethod = do let paramKeys = case invMethod of InvRead _ -> S.fromList $ fst <$> qsParams' diff --git a/src/PostgREST/Query.hs b/src/PostgREST/Query.hs index ae7d1288787..b00ea91f232 100644 --- a/src/PostgREST/Query.hs +++ b/src/PostgREST/Query.hs @@ -1,11 +1,11 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE RecordWildCards #-} module PostgREST.Query - ( openApiQuery - , actionQuery + ( actionQuery , setPgLocals , runPreReq , DbHandler + , QueryResult (..) ) where import qualified Data.Aeson as JSON @@ -41,6 +41,7 @@ import PostgREST.Config.PgVersion (PgVersion (..)) import PostgREST.Error (Error) import PostgREST.MediaType (MediaType (..)) import PostgREST.Plan (ActionPlan (..), + DbActionPlan (..), InspectPlan (..)) import PostgREST.Plan.MutatePlan (MutatePlan (..)) import PostgREST.Plan.ReadPlan (ReadPlanTree) @@ -59,9 +60,13 @@ import Protolude hiding (Handler) type DbHandler = ExceptT Error SQL.Transaction -actionQuery :: ActionPlan -> AppConfig -> ApiRequest -> PgVersion -> DbHandler ResultSet +data QueryResult + = DbResult DbActionPlan ResultSet + | MaybeDbResult InspectPlan (Maybe (TablesMap, RoutineMap, Maybe Text)) -actionQuery WrappedReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ = do +actionQuery :: ActionPlan -> AppConfig -> ApiRequest -> PgVersion -> SchemaCache -> DbHandler QueryResult + +actionQuery (Db plan@WrappedReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} _ _ = do let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan resultSet <- lift . SQL.statement mempty $ @@ -79,37 +84,37 @@ actionQuery WrappedReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreference configDbPreparedStatements failNotSingular wrMedia resultSet optionalRollback conf apiReq - resultSetWTotal conf apiReq resultSet countQuery + DbResult plan <$> resultSetWTotal conf apiReq resultSet countQuery -actionQuery MutateReadPlan{mrMutation=MutationCreate, ..} conf apiReq _ = do +actionQuery (Db plan@MutateReadPlan{mrMutation=MutationCreate, ..}) conf apiReq _ _ = do resultSet <- writeQuery mrReadPlan mrMutatePlan mrMedia mrHandler apiReq conf failNotSingular mrMedia resultSet optionalRollback conf apiReq - pure resultSet + pure $ DbResult plan resultSet -actionQuery MutateReadPlan{mrMutation=MutationUpdate, ..} conf apiReq@ApiRequest{iPreferences=Preferences{..}, ..} _ = do +actionQuery (Db plan@MutateReadPlan{mrMutation=MutationUpdate, ..}) conf apiReq@ApiRequest{iPreferences=Preferences{..}, ..} _ _ = do resultSet <- writeQuery mrReadPlan mrMutatePlan mrMedia mrHandler apiReq conf failNotSingular mrMedia resultSet failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet failsChangesOffLimits (RangeQuery.rangeLimit iTopLevelRange) resultSet optionalRollback conf apiReq - pure resultSet + pure $ DbResult plan resultSet -actionQuery MutateReadPlan{mrMutation=MutationSingleUpsert, ..} conf apiReq _ = do +actionQuery (Db plan@MutateReadPlan{mrMutation=MutationSingleUpsert, ..}) conf apiReq _ _ = do resultSet <- writeQuery mrReadPlan mrMutatePlan mrMedia mrHandler apiReq conf failPut resultSet optionalRollback conf apiReq - pure resultSet + pure $ DbResult plan resultSet -actionQuery MutateReadPlan{mrMutation=MutationDelete, ..} conf apiReq@ApiRequest{iPreferences=Preferences{..}, ..} _ = do +actionQuery (Db plan@MutateReadPlan{mrMutation=MutationDelete, ..}) conf apiReq@ApiRequest{iPreferences=Preferences{..}, ..} _ _ = do resultSet <- writeQuery mrReadPlan mrMutatePlan mrMedia mrHandler apiReq conf failNotSingular mrMedia resultSet failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet failsChangesOffLimits (RangeQuery.rangeLimit iTopLevelRange) resultSet optionalRollback conf apiReq - pure resultSet + pure $ DbResult plan resultSet -actionQuery CallReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} pgVer = do +actionQuery (Db plan@CallReadPlan{..}) conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} pgVer _ = do resultSet <- lift . SQL.statement mempty $ Statements.prepareCall @@ -125,25 +130,23 @@ actionQuery CallReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=P optionalRollback conf apiReq failNotSingular crMedia resultSet failExceedsMaxAffectedPref (preferMaxAffected,preferHandling) resultSet - pure resultSet + pure $ DbResult plan resultSet -openApiQuery :: InspectPlan -> AppConfig -> SchemaCache -> PgVersion -> DbHandler (Maybe (TablesMap, RoutineMap, Maybe Text)) -openApiQuery InspectPlan{ipSchema=tSchema} AppConfig{..} sCache pgVer = +actionQuery (MaybeDb plan@InspectPlan{ipSchema=tSchema}) AppConfig{..} _ pgVer sCache = lift $ case configOpenApiMode of OAFollowPriv -> do tableAccess <- SQL.statement [tSchema] (SchemaCache.accessibleTables pgVer configDbPreparedStatements) - Just <$> ((,,) + MaybeDbResult plan . Just <$> ((,,) (HM.filterWithKey (\qi _ -> S.member qi tableAccess) $ SchemaCache.dbTables sCache) <$> SQL.statement tSchema (SchemaCache.accessibleFuncs pgVer configDbPreparedStatements) <*> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements)) OAIgnorePriv -> - Just <$> ((,,) + MaybeDbResult plan . Just <$> ((,,) (HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch == tSchema) $ SchemaCache.dbTables sCache) (HM.filterWithKey (\(QualifiedIdentifier sch _) _ -> sch == tSchema) $ SchemaCache.dbRoutines sCache) <$> SQL.statement tSchema (SchemaCache.schemaDescription configDbPreparedStatements)) OADisabled -> - pure Nothing - + pure $ MaybeDbResult plan Nothing writeQuery :: ReadPlanTree -> MutatePlan -> MediaType -> MediaHandler -> ApiRequest -> AppConfig -> DbHandler ResultSet writeQuery readPlan mutatePlan mType mHandler ApiRequest{iPreferences=Preferences{..}} conf = diff --git a/src/PostgREST/Response.hs b/src/PostgREST/Response.hs index da715c8e555..12ff370d991 100644 --- a/src/PostgREST/Response.hs +++ b/src/PostgREST/Response.hs @@ -8,7 +8,6 @@ module PostgREST.Response ( infoIdentResponse , infoProcResponse , infoRootResponse - , openApiResponse , actionResponse , PgrstResponse(..) ) where @@ -39,17 +38,18 @@ import PostgREST.ApiRequest.Preferences (PreferRepresentation (..), import PostgREST.ApiRequest.QueryParams (QueryParams (..)) import PostgREST.Config (AppConfig (..)) import PostgREST.MediaType (MediaType (..)) -import PostgREST.Plan (ActionPlan (..), +import PostgREST.Plan (DbActionPlan (..), InspectPlan (..)) import PostgREST.Plan.MutatePlan (MutatePlan (..)) +import PostgREST.Query (QueryResult (..)) import PostgREST.Query.Statements (ResultSet (..)) import PostgREST.Response.GucHeader (GucHeader, unwrapGucHeader) import PostgREST.SchemaCache (SchemaCache (..)) import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..), Schema) import PostgREST.SchemaCache.Routine (FuncVolatility (..), - Routine (..), RoutineMap) -import PostgREST.SchemaCache.Table (Table (..), TablesMap) + Routine (..)) +import PostgREST.SchemaCache.Table (Table (..)) import qualified PostgREST.ApiRequest.Types as ApiRequestTypes import qualified PostgREST.SchemaCache.Routine as Routine @@ -63,9 +63,9 @@ data PgrstResponse = PgrstResponse { , pgrstBody :: LBS.ByteString } -actionResponse :: ActionPlan -> QualifiedIdentifier -> ApiRequest -> ResultSet -> Either Error.Error PgrstResponse +actionResponse :: QueryResult -> QualifiedIdentifier -> ApiRequest -> (Text, Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Either Error.Error PgrstResponse -actionResponse WrappedReadPlan{wrMedia, wrHdrsOnly=headersOnly} identifier ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} resultSet = +actionResponse (DbResult WrappedReadPlan{wrMedia, wrHdrsOnly=headersOnly} resultSet) identifier ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} _ _ _ _ _ = case resultSet of RSStandard{..} -> do let @@ -94,7 +94,7 @@ actionResponse WrappedReadPlan{wrMedia, wrHdrsOnly=headersOnly} identifier ctxAp RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders wrMedia ctxApiRequest) $ LBS.fromStrict plan -actionResponse MutateReadPlan{mrMutation=MutationCreate, mrMutatePlan, mrMedia} QualifiedIdentifier{..} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} resultSet = case resultSet of +actionResponse (DbResult MutateReadPlan{mrMutation=MutationCreate, mrMutatePlan, mrMedia} resultSet) QualifiedIdentifier{..} ctxApiRequest@ApiRequest{iPreferences=Preferences{..}, ..} _ _ _ _ _ = case resultSet of RSStandard{..} -> do let pkCols = case mrMutatePlan of { Insert{insPkCols} -> insPkCols; _ -> mempty;} @@ -134,7 +134,7 @@ actionResponse MutateReadPlan{mrMutation=MutationCreate, mrMutatePlan, mrMedia} RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan -actionResponse MutateReadPlan{mrMutation=MutationUpdate, mrMedia} _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of +actionResponse (DbResult MutateReadPlan{mrMutation=MutationUpdate, mrMedia} resultSet) _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of RSStandard{..} -> do let contentRangeHeader = @@ -156,7 +156,7 @@ actionResponse MutateReadPlan{mrMutation=MutationUpdate, mrMedia} _ ctxApiReques RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan -actionResponse MutateReadPlan{mrMutation=MutationSingleUpsert, mrMedia} _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of +actionResponse (DbResult MutateReadPlan{mrMutation=MutationSingleUpsert, mrMedia} resultSet) _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of RSStandard {..} -> do let prefHeader = maybeToList . prefAppliedHeader $ Preferences Nothing preferRepresentation Nothing preferCount preferTransaction Nothing preferHandling preferTimezone Nothing [] @@ -176,7 +176,7 @@ actionResponse MutateReadPlan{mrMutation=MutationSingleUpsert, mrMedia} _ ctxApi RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan -actionResponse MutateReadPlan{mrMutation=MutationDelete, mrMedia} _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} resultSet = case resultSet of +actionResponse (DbResult MutateReadPlan{mrMutation=MutationDelete, mrMedia} resultSet) _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..}} _ _ _ _ _ = case resultSet of RSStandard {..} -> do let contentRangeHeader = @@ -198,7 +198,7 @@ actionResponse MutateReadPlan{mrMutation=MutationDelete, mrMedia} _ ctxApiReques RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders mrMedia ctxApiRequest) $ LBS.fromStrict plan -actionResponse CallReadPlan{crMedia, crInvMthd=invMethod, crProc=proc} _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} resultSet = case resultSet of +actionResponse (DbResult CallReadPlan{crMedia, crInvMthd=invMethod, crProc=proc} resultSet) _ ctxApiRequest@ApiRequest{iPreferences=Preferences{..},..} _ _ _ _ _ = case resultSet of RSStandard {..} -> do let (status, contentRange) = @@ -225,6 +225,11 @@ actionResponse CallReadPlan{crMedia, crInvMthd=invMethod, crProc=proc} _ ctxApiR RSPlan plan -> Right $ PgrstResponse HTTP.status200 (contentTypeHeaders crMedia ctxApiRequest) $ LBS.fromStrict plan +actionResponse (MaybeDbResult InspectPlan{ipHdrsOnly=headersOnly} body) _ _ versions conf sCache schema negotiatedByProfile = + Right $ PgrstResponse HTTP.status200 + (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile)) + (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body) + infoIdentResponse :: QualifiedIdentifier -> SchemaCache -> Either Error.Error PgrstResponse infoIdentResponse identifier sCache = do @@ -253,12 +258,6 @@ respondInfo allowHeader = let allOrigins = ("Access-Control-Allow-Origin", "*") in Right $ PgrstResponse HTTP.status200 [allOrigins, (HTTP.hAllow, allowHeader)] mempty -openApiResponse :: InspectPlan -> (Text, Text) -> Maybe (TablesMap, RoutineMap, Maybe Text) -> AppConfig -> SchemaCache -> Schema -> Bool -> Either Error.Error PgrstResponse -openApiResponse InspectPlan{ipHdrsOnly=headersOnly} versions body conf sCache schema negotiatedByProfile = - Right $ PgrstResponse HTTP.status200 - (MediaType.toContentType MTOpenAPI : maybeToList (profileHeader schema negotiatedByProfile)) - (maybe mempty (\(x, y, z) -> if headersOnly then mempty else OpenAPI.encode versions conf sCache x y z) body) - -- Status and headers can be overridden as per https://postgrest.org/en/stable/references/transactions.html#response-headers overrideStatusHeaders :: Maybe Text -> Maybe BS.ByteString -> HTTP.Status -> [HTTP.Header]-> Either Error.Error (HTTP.Status, [HTTP.Header]) overrideStatusHeaders rsGucStatus rsGucHeaders pgrstStatus pgrstHeaders = do From 06ff56d323caf37d46f47eb4932f2008b34d5b7f Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 28 Mar 2024 12:18:11 -0500 Subject: [PATCH 2/5] refactor: move isolation/settings logic to Plan.hs --- src/PostgREST/App.hs | 22 ++++------------------ src/PostgREST/Plan.hs | 16 ++++++++++++---- src/PostgREST/Query.hs | 13 ++++++++----- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/PostgREST/App.hs b/src/PostgREST/App.hs index ebd288fef0c..3c13aff8c1d 100644 --- a/src/PostgREST/App.hs +++ b/src/PostgREST/App.hs @@ -9,7 +9,6 @@ Some of its functionality includes: - Producing HTTP Headers according to RFCs. - Content Negotiation -} -{-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE RecordWildCards #-} module PostgREST.App ( postgrest @@ -24,7 +23,6 @@ import Data.String (IsString (..)) import Network.Wai.Handler.Warp (defaultSettings, setHost, setPort, setServerName) -import qualified Data.HashMap.Strict as HM import qualified Data.Text.Encoding as T import qualified Hasql.Transaction.Sessions as SQL import qualified Network.Wai as Wai @@ -54,7 +52,6 @@ import PostgREST.Query (DbHandler) import PostgREST.Response.Performance (ServerTiming (..), serverTimingHeader) import PostgREST.SchemaCache (SchemaCache (..)) -import PostgREST.SchemaCache.Routine (Routine (..)) import PostgREST.Version (docsVersion, prettyVersion) import qualified Data.ByteString.Char8 as BS @@ -172,7 +169,10 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A case iAction of ActDb dbAct -> do (planTime', plan) <- withTiming $ liftEither $ Plan.actionPlan dbAct conf apiReq sCache - (txTime', queryResult) <- withTiming $ runQuery (planIsoLvl plan) (planFunSettings plan) (Plan.actionPlanTxMode plan) $ Query.actionQuery plan conf apiReq pgVer sCache + (txTime', queryResult) <- withTiming $ runDbHandler appState conf (Plan.planIsoLvl conf authRole plan) (Plan.planTxMode plan) authenticated prepared observer $ do + Query.setPgLocals plan conf authClaims authRole apiReq + Query.runPreReq conf + Query.actionQuery plan conf apiReq pgVer sCache (respTime', pgrst) <- withTiming $ liftEither $ Response.actionResponse queryResult (dbActQi dbAct) apiReq (T.decodeUtf8 prettyVersion, docsVersion) conf sCache iSchema iNegotiatedByProfile return $ pgrstResponse (ServerTiming jwtTime parseTime planTime' txTime' respTime') pgrst @@ -190,20 +190,6 @@ handleRequest AuthResult{..} conf appState authenticated prepared pgVer apiReq@A return $ pgrstResponse (ServerTiming jwtTime parseTime Nothing Nothing respTime') pgrst where - roleSettings = fromMaybe mempty (HM.lookup authRole $ configRoleSettings conf) - roleIsoLvl = HM.findWithDefault SQL.ReadCommitted authRole $ configRoleIsoLvl conf - runQuery isoLvl funcSets mode query = - runDbHandler appState conf isoLvl mode authenticated prepared observer $ do - Query.setPgLocals conf authClaims authRole (HM.toList roleSettings) funcSets apiReq - Query.runPreReq conf - query - - planIsoLvl (Plan.Db Plan.CallReadPlan{crProc}) = fromMaybe roleIsoLvl $ pdIsoLvl crProc - planIsoLvl _ = roleIsoLvl - - planFunSettings (Plan.Db Plan.CallReadPlan{crProc}) = pdFuncSettings crProc - planFunSettings _ = mempty - pgrstResponse :: ServerTiming -> Response.PgrstResponse -> Wai.Response pgrstResponse timing (Response.PgrstResponse st hdrs bod) = Wai.responseLBS st (hdrs ++ ([serverTimingHeader timing | configServerTimingEnabled conf])) bod diff --git a/src/PostgREST/Plan.hs b/src/PostgREST/Plan.hs index 25d3013d0fa..69ad1782313 100644 --- a/src/PostgREST/Plan.hs +++ b/src/PostgREST/Plan.hs @@ -22,7 +22,8 @@ module PostgREST.Plan , InspectPlan(..) , inspectPlan , callReadPlan - , actionPlanTxMode + , planTxMode + , planIsoLvl ) where import qualified Data.ByteString.Lazy as LBS @@ -127,9 +128,16 @@ data InspectPlan = InspectPlan { data ActionPlan = Db DbActionPlan | MaybeDb InspectPlan -actionPlanTxMode :: ActionPlan -> SQL.Mode -actionPlanTxMode (Db x) = pTxMode x -actionPlanTxMode (MaybeDb x) = ipTxmode x +planTxMode :: ActionPlan -> SQL.Mode +planTxMode (Db x) = pTxMode x +planTxMode (MaybeDb x) = ipTxmode x + +planIsoLvl :: AppConfig -> ByteString -> ActionPlan -> SQL.IsolationLevel +planIsoLvl AppConfig{configRoleIsoLvl} role actPlan = case actPlan of + Db CallReadPlan{crProc} -> fromMaybe roleIsoLvl $ pdIsoLvl crProc + _ -> roleIsoLvl + where + roleIsoLvl = HM.findWithDefault SQL.ReadCommitted role configRoleIsoLvl actionPlan :: DbAction -> AppConfig -> ApiRequest -> SchemaCache -> Either Error ActionPlan actionPlan dbAct conf apiReq sCache = case dbAct of diff --git a/src/PostgREST/Query.hs b/src/PostgREST/Query.hs index b00ea91f232..dee6f9c270d 100644 --- a/src/PostgREST/Query.hs +++ b/src/PostgREST/Query.hs @@ -53,7 +53,8 @@ import PostgREST.Query.SqlFragment (escapeIdentList, fromQi, import PostgREST.Query.Statements (ResultSet (..)) import PostgREST.SchemaCache (SchemaCache (..)) import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..)) -import PostgREST.SchemaCache.Routine (MediaHandler, RoutineMap) +import PostgREST.SchemaCache.Routine (MediaHandler, Routine (..), + RoutineMap) import PostgREST.SchemaCache.Table (TablesMap) import Protolude hiding (Handler) @@ -238,9 +239,8 @@ optionalRollback AppConfig{..} ApiRequest{iPreferences=Preferences{..}} = do preferTransaction == Just Rollback -- | Set transaction scoped settings -setPgLocals :: AppConfig -> KM.KeyMap JSON.Value -> BS.ByteString -> [(ByteString, ByteString)] -> - [(Text,Text)] -> ApiRequest -> DbHandler () -setPgLocals AppConfig{..} claims role roleSettings funcSettings ApiRequest{..} = lift $ +setPgLocals :: ActionPlan -> AppConfig -> KM.KeyMap JSON.Value -> BS.ByteString -> ApiRequest -> DbHandler () +setPgLocals actPlan AppConfig{..} claims role ApiRequest{..} = lift $ SQL.statement mempty $ SQL.dynamicallyParameterized -- To ensure `GRANT SET ON PARAMETER TO authenticator` works, the role settings must be set before the impersonated role. -- Otherwise the GRANT SET would have to be applied to the impersonated role. See https://github.com/PostgREST/postgrest/issues/3045 @@ -253,13 +253,16 @@ setPgLocals AppConfig{..} claims role roleSettings funcSettings ApiRequest{..} = cookiesSql = setConfigWithConstantNameJSON "request.cookies" iCookies claimsSql = [setConfigWithConstantName ("request.jwt.claims", LBS.toStrict $ JSON.encode claims)] roleSql = [setConfigWithConstantName ("role", role)] - roleSettingsSql = setConfigWithDynamicName <$> roleSettings + roleSettingsSql = setConfigWithDynamicName <$> HM.toList (fromMaybe mempty $ HM.lookup role configRoleSettings) appSettingsSql = setConfigWithDynamicName <$> (join bimap toUtf8 <$> configAppSettings) timezoneSql = maybe mempty (\(PreferTimezone tz) -> [setConfigWithConstantName ("timezone", tz)]) $ preferTimezone iPreferences funcSettingsSql = setConfigWithDynamicName <$> (join bimap toUtf8 <$> funcSettings) searchPathSql = let schemas = escapeIdentList (iSchema : configDbExtraSearchPath) in setConfigWithConstantName ("search_path", schemas) + funcSettings = case actPlan of + Db CallReadPlan{crProc} -> pdFuncSettings crProc + _ -> mempty -- | Runs the pre-request function. runPreReq :: AppConfig -> DbHandler () From 6de3ba543d8e23962e45d7e2c32bb9ed86d70489 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 31 Mar 2024 21:47:37 +0200 Subject: [PATCH 3/5] nix: Use minimal set of texlive dependencies for docs-render --- nix/tools/docs.nix | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/nix/tools/docs.nix b/nix/tools/docs.nix index 2b08e526f57..b015dfbb0c8 100644 --- a/nix/tools/docs.nix +++ b/nix/tools/docs.nix @@ -59,6 +59,24 @@ let ''; render = + let + pdflatex = texlive.combine { + inherit (texlive) + amsmath + booktabs + cancel + gensymb + mathdots + multirow + pgf + pgf-blur + scheme-basic + siunitx + standalone + yhmath + ; + }; + in checkedShellScript { name = "postgrest-docs-render"; @@ -67,7 +85,7 @@ let withTmpDir = true; } '' - ${texlive.combined.scheme-full}/bin/pdflatex -halt-on-error -output-directory="$tmpdir" db.tex + ${pdflatex}/bin/pdflatex -halt-on-error -output-directory="$tmpdir" db.tex ${imagemagick}/bin/convert -density 300 "$tmpdir/db.pdf" ../_static/db.png ''; From 2543b8d724f38c9ee6d122933f338c78c65607f5 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Fri, 29 Mar 2024 15:06:22 -0500 Subject: [PATCH 4/5] fix: clarify PGRST204 error message --- CHANGELOG.md | 1 + src/PostgREST/Error.hs | 2 +- test/spec/Feature/Query/InsertSpec.hs | 6 +++--- test/spec/Feature/Query/UpdateSpec.hs | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cddd98c048..20e3c60d8a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). - #3327, Fix slow responses on schema cache reloads - @steve-chavez - #3340, Log when the LISTEN channel gets a notification - @steve-chavez - #3345, Fix in-database configuration values not loading for `pgrst.server_trace_header` and `pgrst.server_cors_allowed_origins` - @laurenceisla + - #3361, Clarify PGRST204(column not found) error message - @steve-chavez ### Deprecated diff --git a/src/PostgREST/Error.hs b/src/PostgREST/Error.hs index 2a7b5cb7301..f5f46bb1173 100644 --- a/src/PostgREST/Error.hs +++ b/src/PostgREST/Error.hs @@ -249,7 +249,7 @@ instance JSON.ToJSON ApiRequestError where (Just "Try renaming the parameters or the function itself in the database so function overloading can be resolved") toJSON (ColumnNotFound relName colName) = toJsonPgrstError - SchemaCacheErrorCode04 ("Column '" <> colName <> "' of relation '" <> relName <> "' does not exist") Nothing Nothing + SchemaCacheErrorCode04 ("Could not find the '" <> colName <> "' column of '" <> relName <> "' in the schema cache") Nothing Nothing -- | -- If no relationship is found then: diff --git a/test/spec/Feature/Query/InsertSpec.hs b/test/spec/Feature/Query/InsertSpec.hs index 36830e9d917..8afed24b389 100644 --- a/test/spec/Feature/Query/InsertSpec.hs +++ b/test/spec/Feature/Query/InsertSpec.hs @@ -469,7 +469,7 @@ spec actualPgVersion = do {"id": 204, "body": "yyy"}, {"id": 205, "body": "zzz"}]|] `shouldRespondWith` - [json|{"code":"PGRST204","details":null,"hint":null,"message":"Column 'helicopter' of relation 'articles' does not exist"} |] + [json|{"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopter' column of 'articles' in the schema cache"} |] { matchStatus = 400 , matchHeaders = [] } @@ -851,7 +851,7 @@ spec actualPgVersion = do request methodPost "/datarep_todos?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")] [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |] `shouldRespondWith` - [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |] + [json| {"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopters' column of 'datarep_todos' in the schema cache"} |] { matchStatus = 400 , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"] } @@ -906,7 +906,7 @@ spec actualPgVersion = do request methodPost "/datarep_todos_computed?columns=id,label_color,helicopters&select=id,name,label_color,due_at" [("Prefer", "return=representation")] [json| {"due_at": "2019-01-03T11:00:00+00", "smth": "here", "label_color": "invalid", "fake_id": 13} |] `shouldRespondWith` - [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |] + [json| {"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopters' column of 'datarep_todos_computed' in the schema cache"} |] { matchStatus = 400 , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"] } diff --git a/test/spec/Feature/Query/UpdateSpec.hs b/test/spec/Feature/Query/UpdateSpec.hs index b82decfa9e3..4f85aaaf122 100644 --- a/test/spec/Feature/Query/UpdateSpec.hs +++ b/test/spec/Feature/Query/UpdateSpec.hs @@ -342,7 +342,7 @@ spec actualPgVersion = do [("Prefer", "return=representation")] [json|{"body": "yyy"}|] `shouldRespondWith` - [json|{"code":"PGRST204","details":null,"hint":null,"message":"Column 'helicopter' of relation 'articles' does not exist"} |] + [json|{"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopter' column of 'articles' in the schema cache"}|] { matchStatus = 400 , matchHeaders = [] } @@ -888,7 +888,7 @@ spec actualPgVersion = do request methodPatch "/datarep_todos?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")] [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |] `shouldRespondWith` - [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos' does not exist","details":null,"hint":null} |] + [json| {"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopters' column of 'datarep_todos' in the schema cache"} |] { matchStatus = 400 , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"] } @@ -978,7 +978,7 @@ spec actualPgVersion = do request methodPatch "/datarep_todos_computed?id=eq.2&columns=label_color,helicopters" [("Prefer", "return=representation")] [json| {"due_at": "2019-01-03T11:00:00Z", "smth": "here", "label_color": "invalid", "fake_id": 13} |] `shouldRespondWith` - [json| {"code":"PGRST204","message":"Column 'helicopters' of relation 'datarep_todos_computed' does not exist","details":null,"hint":null} |] + [json| {"code":"PGRST204","details":null,"hint":null,"message":"Could not find the 'helicopters' column of 'datarep_todos_computed' in the schema cache"} |] { matchStatus = 400 , matchHeaders = ["Content-Type" <:> "application/json; charset=utf-8"] } From d7c64a93f8ec2ce4cb3ee25617327fc89e30f0ac Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Mon, 1 Apr 2024 12:53:22 -0500 Subject: [PATCH 5/5] docs: update schema cache --- docs/references/errors.rst | 2 +- docs/references/schema_cache.rst | 53 ++++++++++---------------------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/docs/references/errors.rst b/docs/references/errors.rst index 047fec7f767..bb05a9dfabf 100644 --- a/docs/references/errors.rst +++ b/docs/references/errors.rst @@ -259,7 +259,7 @@ Related to the HTTP request elements. Group 2 - Schema Cache ~~~~~~~~~~~~~~~~~~~~~~ -Related to a :ref:`stale schema cache `. Most of the time, these errors are solved by :ref:`reloading the schema cache `. +Related to a :ref:`schema_cache`. Most of the time, these errors are solved by :ref:`schema_reloading`. +---------------+-------------+-------------------------------------------------------------+ | Code | HTTP status | Description | diff --git a/docs/references/schema_cache.rst b/docs/references/schema_cache.rst index 27b4e94ca3b..a46c06c10ba 100644 --- a/docs/references/schema_cache.rst +++ b/docs/references/schema_cache.rst @@ -3,46 +3,29 @@ Schema Cache ============ -Some PostgREST features need metadata from the database schema. Getting this metadata requires expensive queries. To avoid repeating this work, PostgREST uses a schema cache. - -+--------------------------------------------+-------------------------------------------------------------------------------+ -| Feature | Required Metadata | -+============================================+===============================================================================+ -| :ref:`resource_embedding` | Foreign key constraints | -+--------------------------------------------+-------------------------------------------------------------------------------+ -| :ref:`Functions ` | Function signature (parameters, return type, volatility and | -| | `overloading `_) | -+--------------------------------------------+-------------------------------------------------------------------------------+ -| :ref:`Upserts ` | Primary keys | -+--------------------------------------------+-------------------------------------------------------------------------------+ -| :ref:`Insertions ` | Primary keys (optional: only if the Location header is requested) | -+--------------------------------------------+-------------------------------------------------------------------------------+ -| :ref:`OPTIONS requests ` | View INSTEAD OF TRIGGERS and primary keys | -+--------------------------------------------+-------------------------------------------------------------------------------+ -| :ref:`open-api` | Table columns, primary keys and foreign keys | -+ +-------------------------------------------------------------------------------+ -| | View columns and INSTEAD OF TRIGGERS | -+ +-------------------------------------------------------------------------------+ -| | Function signature | -+--------------------------------------------+-------------------------------------------------------------------------------+ - -.. _stale_schema: - -Stale Schema Cache ------------------- - -One operational problem that comes with a cache is that it can go stale. This can happen for PostgREST when you make changes to the metadata before mentioned. Requests that depend on the metadata will fail. - -You can solve this by reloading the cache manually or automatically. +PostgREST requires metadata from the database schema to provide a REST API that abstracts SQL details. One example of this is the interface for :ref:`resource_embedding`. -.. note:: - If you are using :ref:`in_db_config`, a schema reload will always :ref:`reload the configuration` as well. +Getting this metadata requires expensive queries. To avoid repeating this work, PostgREST uses a schema cache. .. _schema_reloading: Schema Cache Reloading ---------------------- +To not let the schema cache go stale (happens when you make changes to the database), you need to reload it. + +You can do this with UNIX signals or with PostgreSQL notifications. It's also possible to do this automatically using `event triggers `_. + +.. note:: + + - If you are using the :ref:`in_db_config`, a schema cache reload will :ref:`reload the configuration` as well. + - There’s no downtime when reloading the schema cache. The reloading will happen on a background thread while serving requests. + +.. _schema_reloading_signals: + +Schema Cache Reloading with Unix Signals +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To manually reload the cache without restarting the PostgREST server, send a SIGUSR1 signal to the server process. .. code:: bash @@ -59,8 +42,6 @@ For docker you can do: # or in docker-compose docker-compose kill -s SIGUSR1 -There’s no downtime when reloading the schema cache. The reloading will happen on a background thread while serving requests. - .. _schema_reloading_notify: Schema Cache Reloading with NOTIFY @@ -81,7 +62,7 @@ The ``pgrst`` notification channel is enabled by default. For configuring the ch Automatic Schema Cache Reloading -------------------------------- -You can do automatic schema cache reloading in a pure SQL way and forget about stale schema cache errors. For this use an `event trigger `_ and ``NOTIFY``. +You can do automatic reloading and forget there is a schema cache. For this use an `event trigger `_ and ``NOTIFY``. .. code-block:: postgres