From 0d5c2f9350a217494a66e3a4a7451340b00413af Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 23 Sep 2023 01:50:23 +0300 Subject: [PATCH 001/180] Adds first commit --- src/backend/distributed/Makefile | 2 +- src/backend/distributed/commands/database.c | 239 ++++++ .../commands/distribute_object_ops.c | 31 + .../distributed/database/database_sharding.h | 53 ++ .../deparser/deparse_database_stmts.c | 208 ++++- .../distributed/sql/citus--12.1-1--12.2-1.sql | 11 + .../sql/downgrades/citus--12.2-1--12.1-1.sql | 1 + src/include/distributed/commands.h | 778 ++++++++---------- src/include/distributed/deparser.h | 3 + src/include/distributed/metadata/distobject.h | 1 + 10 files changed, 887 insertions(+), 440 deletions(-) create mode 100644 src/backend/distributed/database/database_sharding.h diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index 8f28b04b025..f4f9ffd867d 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -18,7 +18,7 @@ generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$( DATA_built = $(generated_sql_files) # directories with source files -SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib shardsplit test transaction utils worker clock +SUBDIRS = . commands connection database ddl deparser executor metadata operations planner progress relay safeclib shardsplit test transaction utils worker clock # enterprise modules SUBDIRS += replication diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 944ff627d4e..62623443422 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -19,6 +19,7 @@ #include "miscadmin.h" #include "nodes/parsenodes.h" #include "utils/syscache.h" +#include "utils/builtins.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" @@ -28,13 +29,42 @@ #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" #include "distributed/worker_transaction.h" +#include "distributed/deparser.h" +#include "distributed/worker_protocol.h" +#include "distributed/metadata/distobject.h" +#include "distributed/database/database_sharding.h" +#include "distributed/deparse_shard_query.h" + + + +/* macros to add DefElems to a list */ +#define DEFELEM_ADD_STRING(options, key, value) { \ + DefElem *elem = makeDefElem(key, (Node *) makeString(value), -1); \ + options = lappend(options, elem); \ +} + +#define DEFELEM_ADD_BOOL(options, key, value) { \ + DefElem *elem = makeDefElem(key, (Node *) makeBoolean(value), -1); \ + options = lappend(options, elem); \ +} + +#define DEFELEM_ADD_INT(options, key, value) { \ + DefElem *elem = makeDefElem(key, (Node *) makeInteger(value), -1); \ + options = lappend(options, elem); \ +} static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); + +static List * CreateDDLTaskList(char *command, List *workerNodeList, + bool outsideTransaction); + +PG_FUNCTION_INFO_V1(citus_internal_database_command); static Oid get_database_owner(Oid db_oid); List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); /* controlled via GUC */ +bool EnableCreateDatabasePropagation = true; bool EnableAlterDatabaseOwner = true; @@ -148,6 +178,78 @@ PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } +/* + * citus_internal_database_command is an internal UDF to + * create/drop a database in an idempotent maner without + * transaction block restrictions. + */ +Datum +citus_internal_database_command(PG_FUNCTION_ARGS) +{ + text *commandText = PG_GETARG_TEXT_P(0); + char *command = text_to_cstring(commandText); + Node *parseTree = ParseTreeNode(command); + + set_config_option("citus.enable_ddl_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + set_config_option("citus.enable_create_database_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + if (IsA(parseTree, CreatedbStmt)) + { + CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); + + bool missingOk = true; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + if (!OidIsValid(databaseOid)) + { + createdb(NULL, (CreatedbStmt *) parseTree); + } + else + { + /* TODO: check database properties */ + } + } + else if (IsA(parseTree, DropdbStmt)) + { + DropdbStmt *stmt = castNode(DropdbStmt, parseTree); + + bool missingOk = true; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + if (!OidIsValid(databaseOid)) + { + /* already dropped? */ + } + else + { + /* remove database from pg_dist_object */ + ObjectAddress dbAddress = { 0 }; + ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); + + if (IsObjectDistributed(&dbAddress)) + { + UnmarkObjectDistributed(&dbAddress); + } + + /* remove database from database shards */ + DeleteDatabaseShardByDatabaseIdLocally(databaseOid); + + DropDatabase(NULL, (DropdbStmt *) parseTree); + } + } + else + { + ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); + } + + PG_RETURN_VOID(); +} + /* * PreprocessAlterDatabaseStmt is executed before the statement is applied to the local @@ -213,6 +315,35 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, #endif +/* + * CreateDDLTaskList creates a task list for running a single DDL command. + */ +static List * +CreateDDLTaskList(char *command, List *workerNodeList, bool outsideTransaction) +{ + List *commandList = list_make3(DISABLE_DDL_PROPAGATION, + command, + ENABLE_DDL_PROPAGATION); + + Task *task = CitusMakeNode(Task); + task->taskType = DDL_TASK; + SetTaskQueryStringList(task, commandList); + task->cannotBeExecutedInTransction = outsideTransaction; + + WorkerNode *workerNode = NULL; + foreach_ptr(workerNode, workerNodeList) + { + ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); + targetPlacement->nodeName = workerNode->workerName; + targetPlacement->nodePort = workerNode->workerPort; + targetPlacement->groupId = workerNode->groupId; + task->taskPlacementList = lappend(task->taskPlacementList, + targetPlacement); + } + + return list_make1(task); +} + /* * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local @@ -242,3 +373,111 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + + +/* + * PostprocessCreatedbStmt creates the plan to synchronize CREATE DATABASE + * across nodes. We use the cannotBeExecutedInTransction option to avoid + * u* sending transaction blocks. + */ +List * +PostprocessCreateDatabaseStmt(Node *node, const char *queryString) +{ + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) + { + return NIL; + } + + CreatedbStmt *stmt = castNode(CreatedbStmt, node); + char *databaseName = stmt->dbname; + bool missingOk = false; + Oid databaseOid = get_database_oid(databaseName, missingOk); + + /* + * TODO: try to reuse regular DDL infrastructure + * + * We do not do this right now because of the AssignDatabaseToShard at the end. + */ + List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); + if (list_length(workerNodes) > 0) + { + char *createDatabaseCommand = DeparseTreeNode(node); + + StringInfo internalCreateCommand = makeStringInfo(); + appendStringInfo(internalCreateCommand, + "SELECT pg_catalog.citus_internal_database_command(%s)", + quote_literal_cstr(createDatabaseCommand)); + + /* + * For the moment, we run CREATE DATABASE in 2PC, though that prevents + * us from immediately doing a pg_dump | pg_restore when dealing with + * a remote template database. + */ + bool outsideTransaction = false; + + List *taskList = CreateDDLTaskList(internalCreateCommand->data, workerNodes, + outsideTransaction); + + bool localExecutionSupported = false; + ExecuteUtilityTaskList(taskList, localExecutionSupported); + } + + /* synchronize pg_dist_object records */ + ObjectAddress dbAddress = { 0 }; + ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); + MarkObjectDistributed(&dbAddress); + + + return NIL; +} + +List * +PreprocessDropDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext) +{ + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) + { + return NIL; + } + + DropdbStmt *stmt = (DropdbStmt *) node; + char *databaseName = stmt->dbname; + bool missingOk = true; + Oid databaseOid = get_database_oid(databaseName, missingOk); + if (databaseOid == InvalidOid) + { + /* let regular ProcessUtility deal with IF NOT EXISTS */ + return NIL; + } + + ObjectAddress dbAddress = { 0 }; + ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); + if (!IsObjectDistributed(&dbAddress)) + { + return NIL; + } + + List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); + if (list_length(workerNodes) == 0) + { + return NIL; + } + + char *dropDatabaseCommand = DeparseTreeNode(node); + + StringInfo internalDropCommand = makeStringInfo(); + appendStringInfo(internalDropCommand, + "SELECT pg_catalog.citus_internal_database_command(%s)", + quote_literal_cstr(dropDatabaseCommand)); + + + /* we execute here to avoid EnsureCoordinator check in ExecuteDistributedDDLJob */ + bool outsideTransaction = false; + List *taskList = CreateDDLTaskList(internalDropCommand->data, workerNodes, + outsideTransaction); + + bool localExecutionSupported = false; + ExecuteUtilityTaskList(taskList, localExecutionSupported); + + return NIL; +} diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index a0a306e8d15..7465a58cd04 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -466,6 +466,28 @@ static DistributeObjectOps Database_Alter = { .markDistributed = false, }; +static DistributeObjectOps Database_Create = { + .deparse = DeparseCreateDatabaseStmt, + .qualify = NULL, + .preprocess = NULL, + .postprocess = PostprocessCreateDatabaseStmt, + .objectType = OBJECT_DATABASE, + .operationType = DIST_OPS_CREATE, + .address = NULL, + .markDistributed = false, +}; + +static DistributeObjectOps Database_Drop = { + .deparse = DeparseDropDatabaseStmt, + .qualify = NULL, + .preprocess = PreprocessDropDatabaseStmt, + .postprocess = NULL, + .objectType = OBJECT_DATABASE, + .operationType = DIST_OPS_DROP, + .address = NULL, + .markDistributed = false, +}; + #if PG_VERSION_NUM >= PG_VERSION_15 static DistributeObjectOps Database_RefreshColl = { .deparse = DeparseAlterDatabaseRefreshCollStmt, @@ -1333,6 +1355,15 @@ GetDistributeObjectOps(Node *node) { return &Database_Alter; } + case T_CreatedbStmt: + { + return &Database_Create; + } + + case T_DropdbStmt: + { + return &Database_Drop; + } #if PG_VERSION_NUM >= PG_VERSION_15 case T_AlterDatabaseRefreshCollStmt: diff --git a/src/backend/distributed/database/database_sharding.h b/src/backend/distributed/database/database_sharding.h new file mode 100644 index 00000000000..390be1383b0 --- /dev/null +++ b/src/backend/distributed/database/database_sharding.h @@ -0,0 +1,53 @@ +/* + * database_sharding.h + * + * Data structure definition for managing backend data and related function + * + * Copyright (c) Microsoft, Inc. + */ + +#ifndef DATABASE_SHARDING_H +#define DATABASE_SHARDING_H + + +#include "tcop/utility.h" + + +/* attributes of citus_catalog.database_shard */ +#define Natts_database_shard 3 +#define Anum_database_shard_database_id 1 +#define Anum_database_shard_node_group_id 2 +#define Anum_database_shard_is_available 3 + + +typedef struct DatabaseShard +{ + /* database oid */ + Oid databaseOid; + + /* node group on which the database shard is placed */ + int nodeGroupId; + + /* whether the database shard is available */ + bool isAvailable; +} DatabaseShard; + +/* citus.enable_database_sharding setting */ +extern bool EnableDatabaseSharding; + +void PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, + ProcessUtilityContext context, + bool *runPreviousUtilityHook); +void PostProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, + ProcessUtilityContext context); +bool DatabaseShardingEnabled(void); +void AssignDatabaseToShard(Oid databaseOid); +void UpdateDatabaseShard(Oid databaseOid, int targetNodeGroupId); +void DeleteDatabaseShardByDatabaseIdLocally(Oid databaseOid); +DatabaseShard * GetDatabaseShardByOid(Oid databaseOid); +List * ListDatabaseShards(void); +int64 CitusDatabaseSize(Oid databaseId); +char * InsertDatabaseShardAssignmentCommand(Oid databaseOid, int nodeGroupId); + + +#endif diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index d3d3ce633ad..1247aba430d 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -24,7 +24,7 @@ #include "distributed/deparser.h" #include "distributed/log_utils.h" #include "parser/parse_type.h" - +#include "distributed/listutils.h" static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); @@ -34,7 +34,7 @@ char * DeparseAlterDatabaseOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); - StringInfoData str = { 0 }; + StringInfoData str = {0}; initStringInfo(&str); Assert(stmt->objectType == OBJECT_DATABASE); @@ -44,7 +44,6 @@ DeparseAlterDatabaseOwnerStmt(Node *node) return str.data; } - static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { @@ -52,18 +51,17 @@ AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) appendStringInfo(buf, "ALTER DATABASE %s OWNER TO %s;", - quote_identifier(strVal((String *) stmt->object)), + quote_identifier(strVal((String *)stmt->object)), RoleSpecString(stmt->newowner, true)); } - static void AppendGrantDatabases(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON DATABASE "); - foreach(cell, stmt->objects) + foreach (cell, stmt->objects) { char *database = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(database)); @@ -74,7 +72,6 @@ AppendGrantDatabases(StringInfo buf, GrantStmt *stmt) } } - static void AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt) { @@ -87,14 +84,12 @@ AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt) AppendGrantSharedSuffix(buf, stmt); } - static void AppendDefElemConnLimit(StringInfo buf, DefElem *def) { - appendStringInfo(buf, " CONNECTION LIMIT %ld", (long int) defGetNumeric(def)); + appendStringInfo(buf, " CONNECTION LIMIT %ld", (long int)defGetNumeric(def)); } - static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) { @@ -104,7 +99,7 @@ AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, "WITH "); - foreach(cell, stmt->options) + foreach (cell, stmt->options) { DefElem *def = castNode(DefElem, lfirst(cell)); if (strcmp(def->defname, "is_template") == 0) @@ -133,14 +128,13 @@ AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) appendStringInfo(buf, ";"); } - char * DeparseGrantOnDatabaseStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_DATABASE); - StringInfoData str = { 0 }; + StringInfoData str = {0}; initStringInfo(&str); AppendGrantOnDatabaseStmt(&str, stmt); @@ -148,13 +142,12 @@ DeparseGrantOnDatabaseStmt(Node *node) return str.data; } - char * DeparseAlterDatabaseStmt(Node *node) { AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); - StringInfoData str = { 0 }; + StringInfoData str = {0}; initStringInfo(&str); AppendAlterDatabaseStmt(&str, stmt); @@ -162,12 +155,11 @@ DeparseAlterDatabaseStmt(Node *node) return str.data; } - #if PG_VERSION_NUM >= PG_VERSION_15 char * DeparseAlterDatabaseRefreshCollStmt(Node *node) { - AlterDatabaseRefreshCollStmt *stmt = (AlterDatabaseRefreshCollStmt *) node; + AlterDatabaseRefreshCollStmt *stmt = (AlterDatabaseRefreshCollStmt *)node; StringInfoData str; initStringInfo(&str); @@ -179,7 +171,6 @@ DeparseAlterDatabaseRefreshCollStmt(Node *node) return str.data; } - #endif static void @@ -192,16 +183,193 @@ AppendAlterDatabaseSetStmt(StringInfo buf, AlterDatabaseSetStmt *stmt) AppendVariableSet(buf, varSetStmt); } - char * DeparseAlterDatabaseSetStmt(Node *node) { AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); - StringInfoData str = { 0 }; + StringInfoData str = {0}; initStringInfo(&str); AppendAlterDatabaseSetStmt(&str, stmt); return str.data; } + +char * +DeparseCreateDatabaseSetStmt(Node *node) +{ + CreatedbStmt *stmt = castNode(CreatedbStmt, node); + StringInfoData str = {0}; + initStringInfo(&str); + + AppendCreatedbStmt(&str, stmt); + + return str.data; +} + +static void +AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) +{ + appendStringInfo(buf, + "CREATE DATABASE %s", + quote_identifier(stmt->dbname)); + + DefElem *option = NULL; + + foreach_ptr(option, stmt->options) + { + if (strcmp(option->defname, "tablespace") == 0) + { + char *tablespaceName = defGetString(option); + + appendStringInfo(buf, " TABLESPACE %s", + quote_identifier(tablespaceName)); + } + else if (strcmp(option->defname, "owner") == 0) + { + char *owner = defGetString(option); + + appendStringInfo(buf, " OWNER %s", + quote_identifier(owner)); + } + else if (strcmp(option->defname, "template") == 0) + { + char *template = defGetString(option); + + appendStringInfo(buf, " TEMPLATE %s", + quote_identifier(template)); + } + else if (strcmp(option->defname, "encoding") == 0) + { + char *encoding = defGetString(option); + + appendStringInfo(buf, " ENCODING %s", + quote_literal_cstr(encoding)); + } + else if (strcmp(option->defname, "locale") == 0) + { + char *locale = defGetString(option); + + appendStringInfo(buf, " LOCALE %s", + quote_literal_cstr(locale)); + } + else if (strcmp(option->defname, "lc_collate") == 0) + { + char *lc_collate = defGetString(option); + + appendStringInfo(buf, " LC_COLLATE %s", + quote_literal_cstr(lc_collate)); + } + else if (strcmp(option->defname, "lc_ctype") == 0) + { + char *lc_ctype = defGetString(option); + + appendStringInfo(buf, " LC_CTYPE %s", + quote_literal_cstr(lc_ctype)); + } + else if (strcmp(option->defname, "icu_locale") == 0) + { + char *icuLocale = defGetString(option); + + appendStringInfo(buf, " ICU_LOCALE %s", + quote_literal_cstr(icuLocale)); + } + else if (strcmp(option->defname, "locale_provider") == 0) + { + char *localeProvider = defGetString(option); + + appendStringInfo(buf, " LOCALE_PROVIDER %s", + quote_literal_cstr(localeProvider)); + } + else if (strcmp(option->defname, "is_template") == 0) + { + bool isTemplate = defGetBoolean(option); + + appendStringInfo(buf, " IS_TEMPLATE %s", + isTemplate ? "true" : "false"); + } + else if (strcmp(option->defname, "allow_connections") == 0) + { + bool allowConnections = defGetBoolean(option); + + appendStringInfo(buf, " ALLOW_CONNECTIONS %s", + allowConnections ? "true" : "false"); + } + else if (strcmp(option->defname, "connection_limit") == 0) + { + int connectionLimit = defGetInt32(option); + + appendStringInfo(buf, " CONNECTION_LIMIT %d", connectionLimit); + } +#if PG_VERSION_NUM >= PG_VERSION_15 + else if (strcmp(option->defname, "collation_version") == 0) + { + char *collationVersion = defGetString(option); + + appendStringInfo(buf, " COLLATION_VERSION %s", + quote_literal_cstr(collationVersion)); + } + else if (strcmp(option->defname, "oid") == 0) + { + Oid objectId = defGetObjectId(option); + + appendStringInfo(buf, " OID %d", objectId); + } + else if (strcmp(option->defname, "strategy") == 0) + { + char *strategy = defGetString(option); + + appendStringInfo(buf, " STRATEGY %s", + quote_literal_cstr(strategy)); + } +#endif + else if (strcmp(option->defname, "location") == 0) + { + /* deprecated option */ + } + else + { + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized CREATE DATABASE option \"%s\"", + option->defname))); + } + } +} + +char * +DeparseDropDatabaseStmt(Node *node) +{ + DropdbStmt *stmt = castNode(DropdbStmt, node); + StringInfoData str = { 0 }; + initStringInfo(&str); + + AppendDropDatabaseStmt(&str, stmt); + + return str.data; +} + + +static void +AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) +{ + appendStringInfo(buf, + "DROP DATABASE %s", + quote_identifier(stmt->dbname)); + + DefElem *option = NULL; + + foreach_ptr(option, stmt->options) + { + if (strcmp(option->defname, "force") == 0) + { + appendStringInfo(buf, " FORCE"); + } + else + { + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized DROP DATABASE option \"%s\"", + option->defname))); + } + } +} diff --git a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql index ec4cc71342d..94814c48be6 100644 --- a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql @@ -1,5 +1,16 @@ -- citus--12.1-1--12.2-1 +/* + * citus_internal_database_command creates a database according to the given command. + */ +CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) + RETURNS void + LANGUAGE C + STRICT +AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; +COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS + 'run a database command without transaction block restrictions'; + -- bump version to 12.2-1 #include "udfs/citus_add_rebalance_strategy/12.2-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql b/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql index 93d121a1207..a8ca092c48c 100644 --- a/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql @@ -1,3 +1,4 @@ -- citus--12.2-1--12.1-1 +DROP FUNCTION pg_catalog.citus_internal_database_command(text); #include "../udfs/citus_add_rebalance_strategy/10.1-1.sql" diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 43429278f1a..6b764dfbb38 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -22,7 +22,6 @@ #include "tcop/utility.h" #include "utils/acl.h" - extern bool AddAllLocalTablesToMetadata; extern bool EnableSchemaBasedSharding; @@ -43,11 +42,11 @@ extern bool EnforceLocalObjectRestrictions; extern void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId, char * - finalRelationName); + finalRelationName); extern void SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid - parentRelationId, + parentRelationId, Oid - partitionRelationId); + partitionRelationId); /* DistOpsOperationType to be used in DistributeObjectOps */ typedef enum DistOpsOperationType @@ -58,7 +57,6 @@ typedef enum DistOpsOperationType DIST_OPS_DROP, } DistOpsOperationType; - /* * DistributeObjectOps specifies handlers for node/object type pairs. * Instances of this type should all be declared in deparse.c. @@ -79,11 +77,11 @@ typedef enum DistOpsOperationType */ typedef struct DistributeObjectOps { - char * (*deparse)(Node *); + char *(*deparse)(Node *); void (*qualify)(Node *); - List * (*preprocess)(Node *, const char *, ProcessUtilityContext); - List * (*postprocess)(Node *, const char *); - List * (*address)(Node *, bool, bool); + List *(*preprocess)(Node *, const char *, ProcessUtilityContext); + List *(*postprocess)(Node *, const char *); + List *(*address)(Node *, bool, bool); bool markDistributed; /* fields used by common implementations, omitted for specialized implementations */ @@ -101,7 +99,7 @@ typedef struct DistributeObjectOps #define CITUS_TRUNCATE_TRIGGER_NAME "citus_truncate_trigger" -const DistributeObjectOps * GetDistributeObjectOps(Node *node); +const DistributeObjectOps *GetDistributeObjectOps(Node *node); /* * Flags that can be passed to GetForeignKeyOids to indicate @@ -140,7 +138,6 @@ typedef enum ExtractForeignKeyConstraintsMode INCLUDE_SINGLE_SHARD_TABLES } ExtractForeignKeyConstraintMode; - /* * Flags that can be passed to GetForeignKeyIdsForColumn to * indicate whether relationId argument should match: @@ -159,7 +156,6 @@ typedef enum SearchForeignKeyColumnFlags /* callers can also pass union of above flags */ } SearchForeignKeyColumnFlags; - typedef enum TenantOperation { TENANT_UNDISTRIBUTE_TABLE = 0, @@ -176,113 +172,101 @@ extern const char *TenantOperationNames[TOTAL_TENANT_OPERATION]; extern void SaveBeginCommandProperties(TransactionStmt *transactionStmt); /* cluster.c - forward declarations */ -extern List * PreprocessClusterStmt(Node *node, const char *clusterCommand, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessClusterStmt(Node *node, const char *clusterCommand, + ProcessUtilityContext processUtilityContext); /* common.c - forward declarations*/ -extern List * PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, - const char *queryString); -extern List * PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString); -extern List * PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, +extern List *PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, + const char *queryString); +extern List *PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, ProcessUtilityContext - processUtilityContext); -extern List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); + processUtilityContext); +extern List *PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString); +extern List *PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); - /* call.c */ extern bool CallDistributedProcedureRemotely(CallStmt *callStmt, DestReceiver *dest); - /* collation.c - forward declarations */ -extern char * CreateCollationDDL(Oid collationId); -extern List * CreateCollationDDLsIdempotent(Oid collationId); -extern List * AlterCollationOwnerObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * RenameCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * AlterCollationSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern char * GenerateBackupNameForCollationCollision(const ObjectAddress *address); -extern List * DefineCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); +extern char *CreateCollationDDL(Oid collationId); +extern List *CreateCollationDDLsIdempotent(Oid collationId); +extern List *AlterCollationOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *RenameCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *AlterCollationSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern char *GenerateBackupNameForCollationCollision(const ObjectAddress *address); +extern List *DefineCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); /* database.c - forward declarations */ -extern List * AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * DatabaseOwnerDDLCommands(const ObjectAddress *address); - -extern List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *DatabaseOwnerDDLCommands(const ObjectAddress *address); -extern List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List *PreprocessAlterDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); -extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessCreateDatabaseStmt(Node *node, const char *queryString); +extern List *PreprocessDropDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); -/* domain.c - forward declarations */ -extern List * CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * DomainRenameConstraintStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern CreateDomainStmt * RecreateDomainStmt(Oid domainOid); + /* domain.c - forward declarations */ + extern List *CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *DomainRenameConstraintStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern CreateDomainStmt *RecreateDomainStmt(Oid domainOid); extern Oid get_constraint_typid(Oid conoid); - /* extension.c - forward declarations */ extern bool IsDropCitusExtensionStmt(Node *parsetree); -extern List * GetDependentFDWsToExtension(Oid extensionId); +extern List *GetDependentFDWsToExtension(Oid extensionId); extern bool IsCreateAlterExtensionUpdateCitusStmt(Node *parsetree); extern void PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree); extern void PreprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern void PostprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern bool ShouldMarkRelationDistributed(Oid relationId); extern void ErrorIfUnstableCreateOrAlterExtensionStmt(Node *parsetree); -extern List * PostprocessCreateExtensionStmt(Node *stmt, const char *queryString); -extern List * PreprocessDropExtensionStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterExtensionSchemaStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PostprocessAlterExtensionSchemaStmt(Node *stmt, - const char *queryString); -extern List * PreprocessAlterExtensionUpdateStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List *PostprocessCreateExtensionStmt(Node *stmt, const char *queryString); +extern List *PreprocessDropExtensionStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterExtensionSchemaStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *PostprocessAlterExtensionSchemaStmt(Node *stmt, + const char *queryString); +extern List *PreprocessAlterExtensionUpdateStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); extern void PostprocessAlterExtensionCitusUpdateStmt(Node *node); -extern List * PreprocessAlterExtensionContentsStmt(Node *node, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * CreateExtensionDDLCommand(const ObjectAddress *extensionAddress); -extern List * AlterExtensionSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * AlterExtensionUpdateStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); +extern List *PreprocessAlterExtensionContentsStmt(Node *node, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *CreateExtensionDDLCommand(const ObjectAddress *extensionAddress); +extern List *AlterExtensionSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *AlterExtensionUpdateStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); extern void CreateExtensionWithVersion(char *extname, char *extVersion); extern void AlterExtensionUpdateStmt(char *extname, char *extVersion); extern int GetExtensionVersionNumber(char *extVersion); @@ -301,21 +285,21 @@ extern void ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(Oid localTableId); extern bool ColumnReferencedByAnyForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid - relationId); -extern List * GetReferencingForeignConstaintCommands(Oid relationOid); -extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId); -extern List * GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId); -extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId); -extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId); -extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags); + relationId); +extern List *GetReferencingForeignConstaintCommands(Oid relationOid); +extern List *GetForeignConstraintToReferenceTablesCommands(Oid relationId); +extern List *GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId); +extern List *GetForeignConstraintToDistributedTablesCommands(Oid relationId); +extern List *GetForeignConstraintFromDistributedTablesCommands(Oid relationId); +extern List *GetForeignConstraintCommandsInternal(Oid relationId, int flags); extern Oid DropFKeysAndUndistributeTable(Oid relationId); extern void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag); -extern List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, - int tableTypeFlag); +extern List *GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, + int tableTypeFlag); extern bool AnyForeignKeyDependsOnIndex(Oid indexId); extern bool HasForeignKeyWithLocalTable(Oid relationId); extern bool HasForeignKeyToReferenceTable(Oid relationOid); -extern List * GetForeignKeysFromLocalTables(Oid relationId); +extern List *GetForeignKeysFromLocalTables(Oid relationId); extern bool TableReferenced(Oid relationOid); extern bool TableReferencing(Oid relationOid); extern bool ConstraintIsAUniquenessConstraint(char *inputConstaintName, Oid relationId); @@ -324,297 +308,269 @@ extern bool ConstraintWithNameIsOfType(char *inputConstaintName, Oid relationId, char targetConstraintType); extern bool ConstraintWithIdIsOfType(Oid constraintId, char targetConstraintType); extern bool TableHasExternalForeignKeys(Oid relationId); -extern List * GetForeignKeyOids(Oid relationId, int flags); +extern List *GetForeignKeyOids(Oid relationId, int flags); extern Oid GetReferencedTableId(Oid foreignKeyId); extern Oid GetReferencingTableId(Oid foreignKeyId); extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId); - /* foreign_data_wrapper.c - forward declarations */ -extern List * PreprocessGrantOnFDWStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern Acl * GetPrivilegesForFDW(Oid FDWOid); - +extern List *PreprocessGrantOnFDWStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern Acl *GetPrivilegesForFDW(Oid FDWOid); /* foreign_server.c - forward declarations */ -extern List * PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool - missing_ok, bool isPostprocess); -extern List * GetForeignServerCreateDDLCommand(Oid serverId); - - -/* foreign_table.c - forward declarations */ -extern List * PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext +extern List *PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *GetForeignServerCreateDDLCommand(Oid serverId); +/* foreign_table.c - forward declarations */ +extern List *PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); /* function.c - forward declarations */ -extern List * PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessCreateFunctionStmt(Node *stmt, - const char *queryString); -extern List * CreateFunctionStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * DefineAggregateStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * PreprocessAlterFunctionStmt(Node *stmt, const char *queryString, +extern List *PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * AlterFunctionStmtObjectAddress(Node *stmt, +extern List *PostprocessCreateFunctionStmt(Node *stmt, + const char *queryString); +extern List *CreateFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * RenameFunctionStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * AlterFunctionOwnerObjectAddress(Node *stmt, +extern List *DefineAggregateStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * AlterFunctionSchemaStmtObjectAddress(Node *stmt, +extern List *PreprocessAlterFunctionStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *AlterFunctionStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *RenameFunctionStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *AlterFunctionOwnerObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *AlterFunctionSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *PreprocessAlterFunctionDependsStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *AlterFunctionDependsStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * PreprocessAlterFunctionDependsStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * AlterFunctionDependsStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); - +extern List *PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); /* grant.c - forward declarations */ -extern List * PreprocessGrantStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessGrantStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void deparsePrivileges(StringInfo privsString, GrantStmt *grantStmt); extern void deparseGrantees(StringInfo granteesString, GrantStmt *grantStmt); - /* index.c - forward declarations */ extern bool IsIndexRenameStmt(RenameStmt *renameStmt); -extern List * PreprocessIndexStmt(Node *createIndexStatement, - const char *createIndexCommand, - ProcessUtilityContext processUtilityContext); -extern char * ChooseIndexName(const char *tabname, Oid namespaceId, - List *colnames, List *exclusionOpNames, - bool primary, bool isconstraint); -extern char * ChooseIndexNameAddition(List *colnames); -extern List * ChooseIndexColumnNames(List *indexElems); +extern List *PreprocessIndexStmt(Node *createIndexStatement, + const char *createIndexCommand, + ProcessUtilityContext processUtilityContext); +extern char *ChooseIndexName(const char *tabname, Oid namespaceId, + List *colnames, List *exclusionOpNames, + bool primary, bool isconstraint); +extern char *ChooseIndexNameAddition(List *colnames); +extern List *ChooseIndexColumnNames(List *indexElems); extern LOCKMODE GetCreateIndexRelationLockMode(IndexStmt *createIndexStatement); -extern List * PreprocessReindexStmt(Node *ReindexStatement, - const char *ReindexCommand, - ProcessUtilityContext processUtilityContext); -extern List * ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * PreprocessDropIndexStmt(Node *dropIndexStatement, - const char *dropIndexCommand, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessIndexStmt(Node *node, - const char *queryString); +extern List *PreprocessReindexStmt(Node *ReindexStatement, + const char *ReindexCommand, + ProcessUtilityContext processUtilityContext); +extern List *ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *PreprocessDropIndexStmt(Node *dropIndexStatement, + const char *dropIndexCommand, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessIndexStmt(Node *node, + const char *queryString); extern void ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement); extern void MarkIndexValid(IndexStmt *indexStmt); -extern List * ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor - pgIndexProcessor, int flags); +extern List *ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcessor, int flags); extern bool IsReindexWithParam_compat(ReindexStmt *stmt, char *paramName); /* objectaddress.c - forward declarations */ -extern List * CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); +extern List *CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); /* owned.c - forward declarations */ -extern List * PreprocessDropOwnedStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessDropOwnedStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); /* policy.c - forward declarations */ -extern List * CreatePolicyCommands(Oid relationId); +extern List *CreatePolicyCommands(Oid relationId); extern void ErrorIfUnsupportedPolicy(Relation relation); extern void ErrorIfUnsupportedPolicyExpr(Node *expr); -extern List * PostprocessCreatePolicyStmt(Node *node, const char *queryString); -extern List * PreprocessAlterPolicyStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessDropPolicyStmt(Node *stmt, const char *queryString, +extern List *PostprocessCreatePolicyStmt(Node *node, const char *queryString); +extern List *PreprocessAlterPolicyStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); +extern List *PreprocessDropPolicyStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); extern bool IsPolicyRenameStmt(RenameStmt *stmt); extern void CreatePolicyEventExtendNames(CreatePolicyStmt *stmt, const char *schemaName, uint64 shardId); extern void AlterPolicyEventExtendNames(AlterPolicyStmt *stmt, const char *schemaName, uint64 shardId); -extern void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 - shardId); -extern void DropPolicyEventExtendNames(DropStmt *stmt, const char *schemaName, uint64 - shardId); +extern void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 shardId); +extern void DropPolicyEventExtendNames(DropStmt *stmt, const char *schemaName, uint64 shardId); extern void AddRangeTableEntryToQueryCompat(ParseState *parseState, Relation relation); /* publication.c - forward declarations */ -extern List * PostProcessCreatePublicationStmt(Node *node, const char *queryString); -extern List * CreatePublicationDDLCommandsIdempotent(const ObjectAddress *address); -extern char * CreatePublicationDDLCommand(Oid publicationId); -extern List * PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityCtx); -extern List * GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd); -extern char * GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, - bool isAdd); -extern List * AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); -extern List * AlterPublicationStmtObjectAddress(Node *node, bool missingOk, +extern List *PostProcessCreatePublicationStmt(Node *node, const char *queryString); +extern List *CreatePublicationDDLCommandsIdempotent(const ObjectAddress *address); +extern char *CreatePublicationDDLCommand(Oid publicationId); +extern List *PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityCtx); +extern List *GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd); +extern char *GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, + bool isAdd); +extern List *AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); +extern List *AlterPublicationStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); +extern List *CreatePublicationStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); +extern List *RenamePublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); -extern List * CreatePublicationStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); -extern List * RenamePublicationStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); /* rename.c - forward declarations*/ -extern List * PreprocessRenameStmt(Node *renameStmt, const char *renameCommand, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessRenameStmt(Node *renameStmt, const char *renameCommand, + ProcessUtilityContext processUtilityContext); extern void ErrorIfUnsupportedRenameStmt(RenameStmt *renameStmt); -extern List * PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); - +extern List *PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); /* role.c - forward declarations*/ -extern List * PostprocessAlterRoleStmt(Node *stmt, const char *queryString); -extern List * PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PostprocessAlterRoleStmt(Node *stmt, const char *queryString); +extern List *PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterRoleRenameStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * GenerateAlterRoleSetCommandForRole(Oid roleid); -extern List * AlterRoleStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List * AlterRoleSetStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List * PreprocessCreateRoleStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessDropRoleStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessGrantRoleStmt(Node *stmt, const char *queryString, +extern List *GenerateAlterRoleSetCommandForRole(Oid roleid); +extern List *AlterRoleStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterRoleSetStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *PreprocessCreateRoleStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * PostprocessGrantRoleStmt(Node *stmt, const char *queryString); -extern List * GenerateCreateOrAlterRoleCommand(Oid roleOid); -extern List * CreateRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); +extern List *PreprocessDropRoleStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessGrantRoleStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessGrantRoleStmt(Node *stmt, const char *queryString); +extern List *GenerateCreateOrAlterRoleCommand(Oid roleOid); +extern List *CreateRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern List * RenameRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern void UnmarkRolesDistributed(List *roles); -extern List * FilterDistributedRoles(List *roles); +extern List *FilterDistributedRoles(List *roles); /* schema.c - forward declarations */ -extern List * PostprocessCreateSchemaStmt(Node *node, const char *queryString); -extern List * PreprocessDropSchemaStmt(Node *dropSchemaStatement, - const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterObjectSchemaStmt(Node *alterObjectSchemaStmt, - const char *alterObjectSchemaCommand); -extern List * PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, - bool isPostprocess); -extern List * AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List *PostprocessCreateSchemaStmt(Node *node, const char *queryString); +extern List *PreprocessDropSchemaStmt(Node *dropSchemaStatement, + const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterObjectSchemaStmt(Node *alterObjectSchemaStmt, + const char *alterObjectSchemaCommand); +extern List *PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, + bool isPostprocess); +extern List *AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); /* sequence.c - forward declarations */ -extern List * PreprocessAlterSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString); -extern List * PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); +extern List *PreprocessAlterSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString); +extern List *PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); #if (PG_VERSION_NUM >= PG_VERSION_15) -extern List * PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); #endif -extern List * PreprocessDropSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * PreprocessRenameSequenceStmt(Node *node, const char *queryString, +extern List *PreprocessDropSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *PreprocessRenameSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessGrantOnSequenceStmt(Node *node, const char *queryString); -extern List * AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List *PostprocessGrantOnSequenceStmt(Node *node, const char *queryString); +extern List *AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); #if (PG_VERSION_NUM >= PG_VERSION_15) -extern List * AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List *AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); #endif -extern List * RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List *RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt); extern void ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt); -extern char * GenerateBackupNameForSequenceCollision(const ObjectAddress *address); +extern char *GenerateBackupNameForSequenceCollision(const ObjectAddress *address); extern void RenameExistingSequenceWithDifferentTypeIfExists(RangeVar *sequence, Oid desiredSeqTypeId); /* statistics.c - forward declarations */ -extern List * PreprocessCreateStatisticsStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessCreateStatisticsStmt(Node *node, const char *queryString); -extern List * CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool - isPostprocess); -extern List * PreprocessDropStatisticsStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * DropStatisticsObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List * PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString); -extern List * AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool - isPostprocess); -extern List * PreprocessAlterStatisticsStmt(Node *node, const char *queryString, +extern List *PreprocessCreateStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, +extern List *PostprocessCreateStatisticsStmt(Node *node, const char *queryString); +extern List *CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); +extern List *PreprocessDropStatisticsStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *DropStatisticsObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, ProcessUtilityContext - processUtilityContext); -extern List * PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString); -extern List * GetExplicitStatisticsCommandList(Oid relationId); -extern List * GetExplicitStatisticsSchemaIdList(Oid relationId); -extern List * GetAlterIndexStatisticsCommands(Oid indexOid); + processUtilityContext); +extern List *PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString); +extern List *AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); +extern List *PreprocessAlterStatisticsStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List *PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString); +extern List *GetExplicitStatisticsCommandList(Oid relationId); +extern List *GetExplicitStatisticsSchemaIdList(Oid relationId); +extern List *GetAlterIndexStatisticsCommands(Oid indexOid); /* subscription.c - forward declarations */ -extern Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); - +extern Node *ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); /* table.c - forward declarations */ -extern List * PreprocessDropTableStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessDropTableStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString); extern bool ShouldEnableLocalReferenceForeignKeys(void); -extern List * PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, - const char *queryString); -extern List * PostprocessAlterTableSchemaStmt(Node *node, const char *queryString); +extern List *PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, + const char *queryString); +extern List *PostprocessAlterTableSchemaStmt(Node *node, const char *queryString); extern void PrepareAlterTableStmtForConstraint(AlterTableStmt *alterTableStatement, Oid relationId, Constraint *constraint); -extern List * PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, +extern List *PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, + ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); +extern List *PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt, bool processLocalRelation); extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt); @@ -626,160 +582,144 @@ extern void ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *c extern void ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, char referencingReplicationModel, Var *distributionColumn, uint32 colocationId); -extern List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, - const char *commandString); -extern List * AlterTableSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * MakeNameListFromRangeVar(const RangeVar *rel); +extern List *InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, + const char *commandString); +extern List *AlterTableSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *MakeNameListFromRangeVar(const RangeVar *rel); extern Oid GetSequenceOid(Oid relationId, AttrNumber attnum); extern bool ConstrTypeUsesIndex(ConstrType constrType); extern bool ConstrTypeCitusCanDefaultName(ConstrType constrType); -extern char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, - char *colname, bool missingTableOk); +extern char *GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, + char *colname, bool missingTableOk); extern void ErrorIfTableHasIdentityColumn(Oid relationId); extern void ConvertNewTableIfNecessary(Node *createStmt); extern void ConvertToTenantTableIfNecessary(AlterObjectSchemaStmt *alterObjectSchemaStmt); /* text_search.c - forward declarations */ -extern List * GetCreateTextSearchConfigStatements(const ObjectAddress *address); -extern List * GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); -extern List * CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address); -extern List * CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address); -extern List * CreateTextSearchConfigurationObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * CreateTextSearchDictObjectAddress(Node *node, +extern List *GetCreateTextSearchConfigStatements(const ObjectAddress *address); +extern List *GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); +extern List *CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address); +extern List *CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address); +extern List *CreateTextSearchConfigurationObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *CreateTextSearchDictObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *RenameTextSearchConfigurationStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *RenameTextSearchDictionaryStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterTextSearchConfigurationStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterTextSearchDictionaryStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *TextSearchConfigurationCommentObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *TextSearchDictCommentObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List * RenameTextSearchConfigurationStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * RenameTextSearchDictionaryStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterTextSearchConfigurationStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterTextSearchDictionaryStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * TextSearchConfigurationCommentObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * TextSearchDictCommentObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List * AlterTextSearchConfigurationOwnerObjectAddress(Node *node, - bool missing_ok, bool - isPostprocess); -extern List * AlterTextSearchDictOwnerObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern char * GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address); -extern char * GenerateBackupNameForTextSearchDict(const ObjectAddress *address); -extern List * get_ts_config_namelist(Oid tsconfigOid); +extern List *AlterTextSearchConfigurationOwnerObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List *AlterTextSearchDictOwnerObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern char *GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address); +extern char *GenerateBackupNameForTextSearchDict(const ObjectAddress *address); +extern List *get_ts_config_namelist(Oid tsconfigOid); /* truncate.c - forward declarations */ extern void PreprocessTruncateStatement(TruncateStmt *truncateStatement); /* type.c - forward declarations */ -extern List * PreprocessRenameTypeAttributeStmt(Node *stmt, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern Node * CreateTypeStmtByObjectAddress(const ObjectAddress *address); -extern List * CompositeTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * CreateEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * AlterTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * AlterEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List * RenameTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * AlterTypeSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List * RenameTypeAttributeStmtObjectAddress(Node *stmt, - bool missing_ok); -extern List * AlterTypeOwnerObjectAddress(Node *stmt, bool missing_ok, bool - isPostprocess); -extern List * CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress); -extern char * GenerateBackupNameForTypeCollision(const ObjectAddress *address); +extern List *PreprocessRenameTypeAttributeStmt(Node *stmt, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern Node *CreateTypeStmtByObjectAddress(const ObjectAddress *address); +extern List *CompositeTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *CreateEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *AlterTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *AlterEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *RenameTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *AlterTypeSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List *RenameTypeAttributeStmtObjectAddress(Node *stmt, + bool missing_ok); +extern List *AlterTypeOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List *CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress); +extern char *GenerateBackupNameForTypeCollision(const ObjectAddress *address); /* function.c - forward declarations */ -extern List * CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress); -extern char * GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); -extern char * GenerateBackupNameForProcCollision(const ObjectAddress *address); -extern ObjectWithArgs * ObjectWithArgsFromOid(Oid funcOid); +extern List *CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress); +extern char *GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); +extern char *GenerateBackupNameForProcCollision(const ObjectAddress *address); +extern ObjectWithArgs *ObjectWithArgsFromOid(Oid funcOid); extern void UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, int *distribution_argument_index, int *colocationId, bool *forceDelegation); /* vacuum.c - forward declarations */ -extern List * PostprocessVacuumStmt(Node *node, const char *vacuumCommand); +extern List *PostprocessVacuumStmt(Node *node, const char *vacuumCommand); /* view.c - forward declarations */ -extern List * PreprocessViewStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessViewStmt(Node *node, const char *queryString); -extern List * ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List * AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List * PreprocessDropViewStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * DropViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern char * CreateViewDDLCommand(Oid viewOid); -extern List * GetViewCreationCommandsOfTable(Oid relationId); -extern char * AlterViewOwnerCommand(Oid viewOid); -extern char * DeparseViewStmt(Node *node); -extern char * DeparseDropViewStmt(Node *node); +extern List *PreprocessViewStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessViewStmt(Node *node, const char *queryString); +extern List *ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *PreprocessDropViewStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *DropViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern char *CreateViewDDLCommand(Oid viewOid); +extern List *GetViewCreationCommandsOfTable(Oid relationId); +extern char *AlterViewOwnerCommand(Oid viewOid); +extern char *DeparseViewStmt(Node *node); +extern char *DeparseDropViewStmt(Node *node); extern bool IsViewDistributed(Oid viewOid); -extern List * CreateViewDDLCommandsIdempotent(Oid viewOid); -extern List * PreprocessAlterViewStmt(Node *node, const char *queryString, +extern List *CreateViewDDLCommandsIdempotent(Oid viewOid); +extern List *PreprocessAlterViewStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessAlterViewStmt(Node *node, const char *queryString); +extern List *PreprocessRenameViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * PostprocessAlterViewStmt(Node *node, const char *queryString); -extern List * PreprocessRenameViewStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessAlterViewSchemaStmt(Node *node, const char *queryString); -extern List * AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List *RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List *PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessAlterViewSchemaStmt(Node *node, const char *queryString); +extern List *AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern bool IsViewRenameStmt(RenameStmt *renameStmt); /* trigger.c - forward declarations */ -extern List * GetExplicitTriggerCommandList(Oid relationId); +extern List *GetExplicitTriggerCommandList(Oid relationId); extern HeapTuple GetTriggerTupleById(Oid triggerId, bool missingOk); -extern List * GetExplicitTriggerIdList(Oid relationId); -extern List * PostprocessCreateTriggerStmt(Node *node, const char *queryString); -extern List * CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool - isPostprocess); +extern List *GetExplicitTriggerIdList(Oid relationId); +extern List *PostprocessCreateTriggerStmt(Node *node, const char *queryString); +extern List *CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern void CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaName, uint64 shardId); -extern List * PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List * PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); +extern List *PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List *PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); extern void AlterTriggerRenameEventExtendNames(RenameStmt *renameTriggerStmt, char *schemaName, uint64 shardId); -extern List * PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString); -extern List * PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List *PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString); +extern List *PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); extern void AlterTriggerDependsEventExtendNames( AlterObjectDependsStmt *alterTriggerDependsStmt, char *schemaName, uint64 shardId); extern void ErrorOutForTriggerIfNotSupported(Oid relationId); extern void ErrorIfRelationHasUnsupportedTrigger(Oid relationId); -extern List * PreprocessDropTriggerStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List *PreprocessDropTriggerStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void DropTriggerEventExtendNames(DropStmt *dropTriggerStmt, char *schemaName, uint64 shardId); -extern List * CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, - const char *queryString); +extern List *CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, + const char *queryString); extern Oid GetTriggerFunctionId(Oid triggerId); /* cascade_table_operation_for_connected_relations.c */ @@ -806,12 +746,12 @@ typedef enum CascadeOperationType extern void CascadeOperationForFkeyConnectedRelations(Oid relationId, LOCKMODE relLockMode, CascadeOperationType - cascadeOperationType); + cascadeOperationType); extern void CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, CascadeOperationType cascadeOperationType); extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList); extern bool RelationIdListHasReferenceTable(List *relationIdList); -extern List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList); +extern List *GetFKeyCreationCommandsForRelationIdList(List *relationIdList); extern void DropRelationForeignKeys(Oid relationId, int flags); extern void SetLocalEnableLocalReferenceForeignKeys(bool state); extern void ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI( @@ -825,7 +765,7 @@ extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConverted); extern bool ShouldAddNewTableToMetadata(Oid relationId); -extern List * GetExplicitIndexOidList(Oid relationId); +extern List *GetExplicitIndexOidList(Oid relationId); extern bool ShouldPropagateSetCommand(VariableSetStmt *setStmt); extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *setCommand); @@ -833,7 +773,7 @@ extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *set extern void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, Oid parentRelationId); extern void UpdateAutoConvertedForConnectedRelations(List *relationId, bool - autoConverted); + autoConverted); /* schema_based_sharding.c */ extern bool ShouldUseSchemaBasedSharding(char *schemaName); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 95d948bc9e9..f7ba8c02b4d 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -227,6 +227,9 @@ extern char * DeparseGrantOnDatabaseStmt(Node *node); extern char * DeparseAlterDatabaseStmt(Node *node); extern char * DeparseAlterDatabaseRefreshCollStmt(Node *node); extern char * DeparseAlterDatabaseSetStmt(Node *node); +extern char * DeparseCreateDatabaseStmt(Node *node); +extern char * DeparseDropDatabaseStmt(Node *node); + /* forward declaration for deparse_publication_stmts.c */ diff --git a/src/include/distributed/metadata/distobject.h b/src/include/distributed/metadata/distobject.h index de56c0e1fb6..ba984091c51 100644 --- a/src/include/distributed/metadata/distobject.h +++ b/src/include/distributed/metadata/distobject.h @@ -21,6 +21,7 @@ extern bool ObjectExists(const ObjectAddress *address); extern bool CitusExtensionObject(const ObjectAddress *objectAddress); extern bool IsAnyObjectDistributed(const List *addresses); +extern bool IsObjectDistributed(const ObjectAddress *address); extern bool ClusterHasDistributedFunctionWithDistArgument(void); extern void MarkObjectDistributed(const ObjectAddress *distAddress); extern void MarkObjectDistributedViaSuperUser(const ObjectAddress *distAddress); From 39cb7977ced66087aab047175bf20e8f231bb489 Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 23 Sep 2023 03:09:06 +0300 Subject: [PATCH 002/180] Fixes additional compile errors --- src/backend/distributed/commands/database.c | 2 + .../distributed/database/database_lock.c | 53 ++ .../distributed/database/database_sharding.c | 544 ++++++++++++++++++ .../distributed/database/database_size.c | 180 ++++++ .../distributed/database/database_sharding.h | 0 .../distributed/pooler/pgbouncer_manager.h | 47 ++ 6 files changed, 826 insertions(+) create mode 100644 src/backend/distributed/database/database_lock.c create mode 100644 src/backend/distributed/database/database_sharding.c create mode 100644 src/backend/distributed/database/database_size.c rename src/{backend => include}/distributed/database/database_sharding.h (100%) create mode 100644 src/include/distributed/pooler/pgbouncer_manager.h diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 62623443422..4886ed62e8b 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -34,6 +34,8 @@ #include "distributed/metadata/distobject.h" #include "distributed/database/database_sharding.h" #include "distributed/deparse_shard_query.h" +#include "distributed/listutils.h" +#include "distributed/adaptive_executor.h" diff --git a/src/backend/distributed/database/database_lock.c b/src/backend/distributed/database/database_lock.c new file mode 100644 index 00000000000..64757832fc2 --- /dev/null +++ b/src/backend/distributed/database/database_lock.c @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * database_lock.c + * Functions for locking a database. + * + * Copyright (c) Microsoft, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "funcapi.h" +#include "fmgr.h" +#include "miscadmin.h" + +#include "catalog/pg_database.h" +#include "commands/dbcommands.h" +#include "distributed/metadata_cache.h" +#include "storage/lmgr.h" + + +static void CitusDatabaseLock(Oid databaseId); + + +PG_FUNCTION_INFO_V1(citus_database_lock_by_name); + + +/* + * citus_database_lock locks the given database in access exclusive mode + * to temporarily block new connections. + */ +Datum +citus_database_lock_by_name(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Name databaseName = PG_GETARG_NAME(0); + bool missingOk = false; + Oid databaseId = get_database_oid(NameStr(*databaseName), missingOk); + + CitusDatabaseLock(databaseId); + + PG_RETURN_VOID(); +} + + +/* + * CitusDatabaseLock locks a database for new connections. + */ +static void +CitusDatabaseLock(Oid databaseId) +{ + LockSharedObject(DatabaseRelationId, databaseId, 0, ExclusiveLock); +} diff --git a/src/backend/distributed/database/database_sharding.c b/src/backend/distributed/database/database_sharding.c new file mode 100644 index 00000000000..b6e844890af --- /dev/null +++ b/src/backend/distributed/database/database_sharding.c @@ -0,0 +1,544 @@ +/*------------------------------------------------------------------------- + * + * database_sharding.c + * + * This file contains module-level definitions. + * + * Copyright (c) 2023, Microsoft, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "fmgr.h" +#include "miscadmin.h" + +#include "citus_version.h" +#include "pg_version_compat.h" + +#include "access/genam.h" +#include "commands/dbcommands.h" +#include "distributed/connection_management.h" +#include "distributed/database/database_sharding.h" +#include "distributed/deparser.h" +#include "distributed/deparse_shard_query.h" +#include "distributed/listutils.h" +#include "distributed/metadata_sync.h" +#include "distributed/remote_commands.h" +#include "distributed/shared_library_init.h" +#include "distributed/worker_transaction.h" +#include "executor/spi.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "postmaster/postmaster.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" + + +static void ExecuteCommandInControlDatabase(char *command); +static void AllowConnectionsOnlyOnNodeGroup(Oid databaseOid, Oid nodeGroupId); +static void InsertDatabaseShardAssignment(Oid databaseOid, int nodeGroupId); +static void InsertDatabaseShardAssignmentLocally(Oid databaseOid, int nodeGroupId); +static void InsertDatabaseShardAssignmentOnOtherNodes(Oid databaseOid, int nodeGroupId); +static void DeleteDatabaseShardByDatabaseId(Oid databaseOid); +static void DeleteDatabaseShardByDatabaseIdOnOtherNodes(Oid databaseOid); +static DatabaseShard * TupleToDatabaseShard(HeapTuple heapTuple, + TupleDesc tupleDescriptor); +static char * DeleteDatabaseShardByDatabaseIdCommand(Oid databaseOid); + + +PG_FUNCTION_INFO_V1(database_shard_assign); +PG_FUNCTION_INFO_V1(citus_internal_add_database_shard); +PG_FUNCTION_INFO_V1(citus_internal_delete_database_shard); + + +/* citus.enable_database_sharding setting */ +bool EnableDatabaseSharding = false; + +/* citus.database_sharding_pgbouncer_file setting */ +char *DatabaseShardingPgBouncerFile = ""; + + +/* + * PreProcessUtilityInDatabaseShard handles DDL commands that occur within a + * database shard and require global coordination: + * - CREATE/ALTER/DROP DATABASE + * - CREATE/ALTER/DROP ROLE/USER/GROUP + */ +void +PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, + ProcessUtilityContext context, + bool *runPreviousUtilityHook) +{ + if (!EnableDatabaseSharding || context != PROCESS_UTILITY_TOPLEVEL) + { + return; + } + + if (EnableCreateDatabasePropagation) + { + if (IsA(parseTree, CreatedbStmt)) + { + char *command = DeparseCreatedbStmt(parseTree); + ExecuteCommandInControlDatabase(command); + + /* command is fully delegated to control database */ + *runPreviousUtilityHook = false; + } + else if (IsA(parseTree, DropdbStmt)) + { + char *command = DeparseDropdbStmt(parseTree); + ExecuteCommandInControlDatabase(command); + + /* command is fully delegated to control database */ + *runPreviousUtilityHook = false; + } + } +} + + +/* + * PostProcessUtilityInDatabaseShard is currently a noop. + */ +void +PostProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, + ProcessUtilityContext context) +{ + if (!EnableDatabaseSharding || context != PROCESS_UTILITY_TOPLEVEL) + { + return; + } +} + + +/* + * ExecuteCommandInControlDatabase connects to localhost to execute a command + * in the main Citus database. + */ +static void +ExecuteCommandInControlDatabase(char *command) +{ + int connectionFlag = FORCE_NEW_CONNECTION; + + MultiConnection *connection = + GetNodeUserDatabaseConnection(connectionFlag, LocalHostName, PostPortNumber, + NULL, CitusMainDatabase); + + ExecuteCriticalRemoteCommand(connection, + "SET application_name TO 'citus_database_shard'"); + ExecuteCriticalRemoteCommand(connection, command); + CloseConnection(connection); +} + + +/* + * database_shard_assign assigns an existing database to a node. + */ +Datum +database_shard_assign(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + char *databaseName = text_to_cstring(PG_GETARG_TEXT_P(0)); + + bool missingOk = false; + Oid databaseOid = get_database_oid(databaseName, missingOk); + + if (!pg_database_ownercheck(databaseOid, GetUserId())) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to assign database \"%s\" " + "to a shard", + databaseName))); + } + + if (GetDatabaseShardByOid(databaseOid) != NULL) + { + ereport(ERROR, (errmsg("database is already assigned to a shard"))); + } + + AssignDatabaseToShard(databaseOid); + + PG_RETURN_VOID(); +} + + +/* + * AssignDatabaseToShard finds a suitable node for the given + * database and assigns it. + */ +void +AssignDatabaseToShard(Oid databaseOid) +{ + int nodeGroupId = GetLocalGroupId(); + + List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); + if (list_length(workerNodes) > 0) + { + /* TODO: actually look for available space */ + int workerNodeIndex = databaseOid % list_length(workerNodes); + WorkerNode *workerNode = list_nth(workerNodes, workerNodeIndex); + nodeGroupId = workerNode->groupId; + } + + InsertDatabaseShardAssignment(databaseOid, nodeGroupId); + AllowConnectionsOnlyOnNodeGroup(databaseOid, nodeGroupId); + + ReconfigurePgBouncersOnCommit = true; +} + + +/* + * AllowConnectionsOnlyOnNodeGroup sets the ALLOW_CONNECTIONS properties on + * the database to false, except on nodeGroupId. + */ +static void +AllowConnectionsOnlyOnNodeGroup(Oid databaseOid, Oid nodeGroupId) +{ + StringInfo command = makeStringInfo(); + char *databaseName = get_database_name(databaseOid); + + List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); + WorkerNode *workerNode = NULL; + + foreach_ptr(workerNode, workerNodes) + { + resetStringInfo(command); + + if (workerNode->groupId == nodeGroupId) + { + appendStringInfo(command, "GRANT CONNECT ON DATABASE %s TO public", + quote_identifier(databaseName)); + } + else + { + appendStringInfo(command, "REVOKE CONNECT ON DATABASE %s FROM public", + quote_identifier(databaseName)); + } + + if (workerNode->groupId == GetLocalGroupId()) + { + ExecuteQueryViaSPI(command->data, SPI_OK_UTILITY); + } + else + { + SendCommandToWorker(workerNode->workerName, workerNode->workerPort, + command->data); + } + } +} + + +/* + * InsertDatabaseShardAssignment inserts a record into the local + * citus_catalog.database_sharding table. + */ +static void +InsertDatabaseShardAssignment(Oid databaseOid, int nodeGroupId) +{ + InsertDatabaseShardAssignmentLocally(databaseOid, nodeGroupId); + + if (EnableMetadataSync) + { + InsertDatabaseShardAssignmentOnOtherNodes(databaseOid, nodeGroupId); + } +} + + +/* + * InsertDatabaseShardAssignmentLocally inserts a record into the local + * citus_catalog.database_sharding table. + */ +static void +InsertDatabaseShardAssignmentLocally(Oid databaseOid, int nodeGroupId) +{ + Datum values[Natts_database_shard]; + bool isNulls[Natts_database_shard]; + + /* form new shard tuple */ + memset(values, 0, sizeof(values)); + memset(isNulls, false, sizeof(isNulls)); + + values[Anum_database_shard_database_id - 1] = ObjectIdGetDatum(databaseOid); + values[Anum_database_shard_node_group_id - 1] = Int32GetDatum(nodeGroupId); + values[Anum_database_shard_is_available - 1] = BoolGetDatum(true); + + /* open shard relation and insert new tuple */ + Relation databaseShardTable = table_open(DatabaseShardRelationId(), RowExclusiveLock); + + TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); + HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); + + CatalogTupleInsert(databaseShardTable, heapTuple); + + CommandCounterIncrement(); + table_close(databaseShardTable, NoLock); +} + + +/* + * InsertDatabaseShardAssignmentOnOtherNodes inserts a record into the + * citus_catalog.database_sharding table on other nodes. + */ +static void +InsertDatabaseShardAssignmentOnOtherNodes(Oid databaseOid, int nodeGroupId) +{ + char *insertCommand = InsertDatabaseShardAssignmentCommand(databaseOid, nodeGroupId); + SendCommandToWorkersWithMetadata(insertCommand); +} + + +/* + * UpdateDatabaseShard updates a database shard after it is moved to a new node. + */ +void +UpdateDatabaseShard(Oid databaseOid, int targetNodeGroupId) +{ + DeleteDatabaseShardByDatabaseId(databaseOid); + InsertDatabaseShardAssignment(databaseOid, targetNodeGroupId); + AllowConnectionsOnlyOnNodeGroup(databaseOid, targetNodeGroupId); +} + + +/* + * DeleteDatabaseShardByDatabaseId deletes a record from the + * citus_catalog.database_sharding table. + */ +static void +DeleteDatabaseShardByDatabaseId(Oid databaseOid) +{ + DeleteDatabaseShardByDatabaseIdLocally(databaseOid); + + if (EnableMetadataSync) + { + DeleteDatabaseShardByDatabaseIdOnOtherNodes(databaseOid); + } +} + + +/* + * DeleteDatabaseShardByDatabaseIdLocally deletes a database_shard record by database OID. + */ +void +DeleteDatabaseShardByDatabaseIdLocally(Oid databaseOid) +{ + Relation databaseShardTable = table_open(DatabaseShardRelationId(), + RowExclusiveLock); + + const int scanKeyCount = 1; + ScanKeyData scanKey[1]; + bool indexOK = true; + + ScanKeyInit(&scanKey[0], Anum_database_shard_database_id, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(databaseOid)); + + SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, + DatabaseShardPrimaryKeyIndexId(), + indexOK, + NULL, scanKeyCount, scanKey); + + HeapTuple heapTuple = systable_getnext(scanDescriptor); + if (heapTuple != NULL) + { + simple_heap_delete(databaseShardTable, &heapTuple->t_self); + } + + systable_endscan(scanDescriptor); + + CommandCounterIncrement(); + table_close(databaseShardTable, NoLock); +} + + +/* + * DeleteDatabaseShardByDatabaseIdOnOtherNodes deletes a record from the + * citus_catalog.database_sharding table on other nodes. + */ +static void +DeleteDatabaseShardByDatabaseIdOnOtherNodes(Oid databaseOid) +{ + char *deleteCommand = DeleteDatabaseShardByDatabaseIdCommand(databaseOid); + SendCommandToWorkersWithMetadata(deleteCommand); +} + + +/* + * ListDatabaseShards lists all database shards in citus_catalog.database_shard. + */ +List * +ListDatabaseShards(void) +{ + Relation databaseShardTable = table_open(DatabaseShardRelationId(), AccessShareLock); + TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); + + List *dbShardList = NIL; + int scanKeyCount = 0; + bool indexOK = false; + + SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, InvalidOid, + indexOK, NULL, scanKeyCount, NULL); + + HeapTuple heapTuple = NULL; + while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) + { + DatabaseShard *dbShard = TupleToDatabaseShard(heapTuple, tupleDescriptor); + dbShardList = lappend(dbShardList, dbShard); + } + + systable_endscan(scanDescriptor); + table_close(databaseShardTable, NoLock); + + return dbShardList; +} + + +/* + * GetDatabaseShardByOid gets a database shard by database OID or + * NULL if no database shard could be found. + */ +DatabaseShard * +GetDatabaseShardByOid(Oid databaseOid) +{ + DatabaseShard *result = NULL; + + Relation databaseShardTable = table_open(DatabaseShardRelationId(), AccessShareLock); + TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); + + const int scanKeyCount = 1; + ScanKeyData scanKey[1]; + bool indexOK = true; + + ScanKeyInit(&scanKey[0], Anum_database_shard_database_id, + BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(databaseOid)); + + SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, + DatabaseShardPrimaryKeyIndexId(), + indexOK, + NULL, scanKeyCount, scanKey); + + HeapTuple heapTuple = systable_getnext(scanDescriptor); + if (HeapTupleIsValid(heapTuple)) + { + result = TupleToDatabaseShard(heapTuple, tupleDescriptor); + } + + systable_endscan(scanDescriptor); + table_close(databaseShardTable, NoLock); + + return result; +} + + +/* + * TupleToDatabaseShard converts a database_shard record tuple into a DatabaseShard struct. + */ +static DatabaseShard * +TupleToDatabaseShard(HeapTuple heapTuple, TupleDesc tupleDescriptor) +{ + Datum datumArray[Natts_database_shard]; + bool isNullArray[Natts_database_shard]; + heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); + + DatabaseShard *record = palloc0(sizeof(DatabaseShard)); + + record->databaseOid = + DatumGetObjectId(datumArray[Anum_database_shard_database_id - 1]); + + record->nodeGroupId = + DatumGetInt32(datumArray[Anum_database_shard_node_group_id - 1]); + + record->isAvailable = + DatumGetBool(datumArray[Anum_database_shard_is_available - 1]); + + return record; +} + + +/* + * citus_internal_add_database_shard is an internal UDF to + * add a row to database_shard. + */ +Datum +citus_internal_add_database_shard(PG_FUNCTION_ARGS) +{ + char *databaseName = TextDatumGetCString(PG_GETARG_DATUM(0)); + int nodeGroupId = PG_GETARG_INT32(1); + + bool missingOk = false; + Oid databaseOid = get_database_oid(databaseName, missingOk); + + if (!pg_database_ownercheck(databaseOid, GetUserId())) + { + aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, + databaseName); + } + + InsertDatabaseShardAssignmentLocally(databaseOid, nodeGroupId); + + + PG_RETURN_VOID(); +} + + +/* + * InsertDatabaseShardAssignmentCommand returns a command to insert a database shard + * assignment into the metadata on a remote node. + */ +char * +InsertDatabaseShardAssignmentCommand(Oid databaseOid, int nodeGroupId) +{ + StringInfo command = makeStringInfo(); + char *databaseName = get_database_name(databaseOid); + + appendStringInfo(command, + "SELECT pg_catalog.citus_internal_add_database_shard(%s,%d)", + quote_literal_cstr(databaseName), + nodeGroupId); + + return command->data; +} + + +/* + * citus_internal_delete_database_shard is an internal UDF to + * delete a row from database_shard. + */ +Datum +citus_internal_delete_database_shard(PG_FUNCTION_ARGS) +{ + char *databaseName = TextDatumGetCString(PG_GETARG_DATUM(0)); + + bool missingOk = false; + Oid databaseOid = get_database_oid(databaseName, missingOk); + + if (!pg_database_ownercheck(databaseOid, GetUserId())) + { + aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, + databaseName); + } + + DeleteDatabaseShardByDatabaseIdLocally(databaseOid); + + + + PG_RETURN_VOID(); +} + + +/* + * DeleteDatabaseShardByDatabaseIdCommand returns a command to delete a database shard + * assignment from the metadata on a remote node. + */ +static char * +DeleteDatabaseShardByDatabaseIdCommand(Oid databaseOid) +{ + StringInfo command = makeStringInfo(); + char *databaseName = get_database_name(databaseOid); + + appendStringInfo(command, + "SELECT pg_catalog.citus_internal_delete_database_shard(%s)", + quote_literal_cstr(databaseName)); + + return command->data; +} diff --git a/src/backend/distributed/database/database_size.c b/src/backend/distributed/database/database_size.c new file mode 100644 index 00000000000..46e72ad13b8 --- /dev/null +++ b/src/backend/distributed/database/database_size.c @@ -0,0 +1,180 @@ +/*------------------------------------------------------------------------- + * + * database_size.c + * Functions for getting the size of a database. + * + * Copyright (c) Microsoft, Inc. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "funcapi.h" +#include "fmgr.h" +#include "miscadmin.h" + +#include "commands/dbcommands.h" +#include "distributed/citus_safe_lib.h" +#include "distributed/database/database_sharding.h" +#include "distributed/listutils.h" +#include "distributed/metadata_cache.h" +#include "distributed/remote_commands.h" +#include "distributed/worker_transaction.h" +#include "utils/builtins.h" + + +static int64 GetLocalDatabaseSize(Oid databaseId); +static int64 CitusDatabaseShardSize(DatabaseShard *dbShard); +static int64 CitusDatabaseSizeOnNodeList(Oid databaseId, List *workerNodeList); + + +PG_FUNCTION_INFO_V1(citus_database_size_oid); +PG_FUNCTION_INFO_V1(citus_database_size_name); + + +/* + * citus_database_size_oid returns the size of a Citus database + * with the given oid. + */ +Datum +citus_database_size_oid(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Oid databaseId = PG_GETARG_OID(0); + int64 size = CitusDatabaseSize(databaseId); + + PG_RETURN_INT64(size); +} + + +/* + * citus_database_size_name returns the size of a Citus database + * with the given name. + */ +Datum +citus_database_size_name(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + + Name databaseName = PG_GETARG_NAME(0); + bool missingOk = false; + Oid databaseId = get_database_oid(NameStr(*databaseName), missingOk); + + int64 size = CitusDatabaseSize(databaseId); + + PG_RETURN_INT64(size); +} + + +/* + * CitusDatabaseSize returns the size of a Citus database. + */ +int64 +CitusDatabaseSize(Oid databaseId) +{ + DatabaseShard *dbShard = GetDatabaseShardByOid(databaseId); + if (dbShard != NULL) + { + /* for known database shards, get the remote size */ + return CitusDatabaseShardSize(dbShard); + } + + if (databaseId == MyDatabaseId) + { + /* for the current database, get the size from all nodes */ + List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); + return CitusDatabaseSizeOnNodeList(databaseId, workerNodes); + } + + /* for other databases, get the local size */ + /* TODO: get it from main database? */ + return GetLocalDatabaseSize(databaseId); +} + + +/* + * GetLocalDatabaseSize returns the local database size by calling pg_database_size. + */ +static int64 +GetLocalDatabaseSize(Oid databaseId) +{ + Datum databaseIdDatum = ObjectIdGetDatum(databaseId); + Datum sizeDatum = DirectFunctionCall1(pg_database_size_oid, databaseIdDatum); + return DatumGetInt64(sizeDatum); +} + + +/* + * CitusDatabaseShardSize gets the database size for a specific + * shard. + */ +static int64 +CitusDatabaseShardSize(DatabaseShard *dbShard) +{ + WorkerNode *workerNode = LookupNodeForGroup(dbShard->nodeGroupId); + + return CitusDatabaseSizeOnNodeList(dbShard->databaseOid, list_make1(workerNode)); +} + + +/* + * CitusDatabaseSizeOnNodeList returns the sum of the sizes + * for a given database from all nodes in the list. + */ +static int64 +CitusDatabaseSizeOnNodeList(Oid databaseId, List *workerNodeList) +{ + int64 size = 0; + + bool raiseInterrupts = true; + + char *databaseName = get_database_name(databaseId); + char *command = psprintf("SELECT pg_catalog.pg_database_size(%s)", + quote_literal_cstr(databaseName)); + + WorkerNode *workerNode = NULL; + foreach_ptr(workerNode, workerNodeList) + { + if (workerNode->groupId == GetLocalGroupId()) + { + return GetLocalDatabaseSize(databaseId); + } + + int connectionFlags = 0; + MultiConnection *connection = GetNodeConnection(connectionFlags, + workerNode->workerName, + workerNode->workerPort); + + int querySent = SendRemoteCommand(connection, command); + if (querySent == 0) + { + ReportConnectionError(connection, ERROR); + } + + PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); + if (!IsResponseOK(result)) + { + ReportResultError(connection, result, ERROR); + } + + if (PQntuples(result) != 1 || PQnfields(result) != 1) + { + PQclear(result); + ClearResults(connection, raiseInterrupts); + + ereport(ERROR, (errmsg("unexpected number of columns returned by: %s", + command))); + } + + if (!PQgetisnull(result, 0, 0)) + { + char *sizeString = PQgetvalue(result, 0, 0); + size += SafeStringToUint64(sizeString); + } + + PQclear(result); + ClearResults(connection, raiseInterrupts); + } + + return size; +} diff --git a/src/backend/distributed/database/database_sharding.h b/src/include/distributed/database/database_sharding.h similarity index 100% rename from src/backend/distributed/database/database_sharding.h rename to src/include/distributed/database/database_sharding.h diff --git a/src/include/distributed/pooler/pgbouncer_manager.h b/src/include/distributed/pooler/pgbouncer_manager.h new file mode 100644 index 00000000000..530d1b1ee39 --- /dev/null +++ b/src/include/distributed/pooler/pgbouncer_manager.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * pgbouncer_manager.h + * Functions for managing outbound pgbouncers + * + * Copyright (c) Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#ifndef PGBOUNCER_MANAGER_H +#define PGBOUNCER_MANAGER_H + +/* default number of inbound pgbouncer processes (0 is disabled) */ +#define PGBOUNCER_INBOUND_PROCS_DEFAULT 0 + +/* bits reserved for local pgbouncer ID in the pgbouncer peer_id */ +#define PGBOUNCER_PEER_ID_LOCAL_ID_BITS 5 + +/* maximum number of inbound pgbouncer processes */ +#define PGBOUNCER_INBOUND_PROCS_MAX ((1 << PGBOUNCER_PEER_ID_LOCAL_ID_BITS) - 1) + + +/* GUC variable that sets the number of inbound pgbouncer procs */ +extern int PgBouncerInboundProcs; + +/* GUC variable that sets the inbound pgbouncer port */ +extern int PgBouncerInboundPort; + +/* GUC variable that sets the path to pgbouncer executable */ +extern char *PgBouncerPath; + +/* GUC variable that sets a pgbouncer file to include */ +extern char *PgBouncerIncludeConfig; + +/* global variable to request pgbouncer reconfiguration */ +extern bool ReconfigurePgBouncersOnCommit; + +void InitializeSharedPgBouncerManager(void); +size_t SharedPgBouncerManagerShmemSize(void); +void PgBouncerManagerMain(Datum arg); +bool PauseDatabaseOnInboundPgBouncers(char *databaseName); +bool ResumeDatabaseOnInboundPgBouncers(char *databaseName); +void TriggerPgBouncerReconfigureIfNeeded(void); + + +#endif /* PGBOUNCER_MANAGER_H */ From bd398a627b99eb10de1286cc8fbc205fe1d0a96a Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 23 Sep 2023 13:28:49 +0300 Subject: [PATCH 003/180] Fixes compile errors --- .../distributed/database/database_sharding.c | 5 +-- .../deparser/deparse_database_stmts.c | 36 ++++++++++--------- src/backend/distributed/metadata/distobject.c | 4 +-- .../distributed/metadata/metadata_cache.c | 30 ++++++++++++++-- src/backend/distributed/shared_library_init.c | 2 ++ .../distributed/commands/utility_hook.h | 1 + src/include/distributed/metadata_cache.h | 2 ++ src/include/distributed/shared_library_init.h | 2 ++ 8 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/database/database_sharding.c b/src/backend/distributed/database/database_sharding.c index b6e844890af..51cd4a8f6f1 100644 --- a/src/backend/distributed/database/database_sharding.c +++ b/src/backend/distributed/database/database_sharding.c @@ -24,6 +24,7 @@ #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/metadata_sync.h" +#include "distributed/pooler/pgbouncer_manager.h" #include "distributed/remote_commands.h" #include "distributed/shared_library_init.h" #include "distributed/worker_transaction.h" @@ -80,7 +81,7 @@ PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, { if (IsA(parseTree, CreatedbStmt)) { - char *command = DeparseCreatedbStmt(parseTree); + char *command = DeparseCreateDatabaseStmt(parseTree); ExecuteCommandInControlDatabase(command); /* command is fully delegated to control database */ @@ -88,7 +89,7 @@ PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, } else if (IsA(parseTree, DropdbStmt)) { - char *command = DeparseDropdbStmt(parseTree); + char *command = DeparseDropDatabaseStmt(parseTree); ExecuteCommandInControlDatabase(command); /* command is fully delegated to control database */ diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 1247aba430d..864a9cdf7b6 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -196,18 +196,6 @@ DeparseAlterDatabaseSetStmt(Node *node) return str.data; } -char * -DeparseCreateDatabaseSetStmt(Node *node) -{ - CreatedbStmt *stmt = castNode(CreatedbStmt, node); - StringInfoData str = {0}; - initStringInfo(&str); - - AppendCreatedbStmt(&str, stmt); - - return str.data; -} - static void AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -338,18 +326,17 @@ AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) } char * -DeparseDropDatabaseStmt(Node *node) +DeparseCreateDatabaseStmt(Node *node) { - DropdbStmt *stmt = castNode(DropdbStmt, node); - StringInfoData str = { 0 }; + CreatedbStmt *stmt = castNode(CreatedbStmt, node); + StringInfoData str = {0}; initStringInfo(&str); - AppendDropDatabaseStmt(&str, stmt); + AppendCreatedbStmt(&str, stmt); return str.data; } - static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { @@ -373,3 +360,18 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) } } } + +char * +DeparseDropDatabaseStmt(Node *node) +{ + DropdbStmt *stmt = castNode(DropdbStmt, node); + StringInfoData str = { 0 }; + initStringInfo(&str); + + AppendDropDatabaseStmt(&str, stmt); + + return str.data; +} + + + diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index c6a8b0a2298..a025ba73fa3 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -53,7 +53,7 @@ static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, Datum *paramValues); -static bool IsObjectDistributed(const ObjectAddress *address); +bool IsObjectDistributed(const ObjectAddress *address); PG_FUNCTION_INFO_V1(citus_unmark_object_distributed); PG_FUNCTION_INFO_V1(master_unmark_object_distributed); @@ -392,7 +392,7 @@ UnmarkObjectDistributed(const ObjectAddress *address) * IsObjectDistributed returns if the object addressed is already distributed in the * cluster. This performs a local indexed lookup in pg_dist_object. */ -static bool +bool IsObjectDistributed(const ObjectAddress *address) { ScanKeyData key[3]; diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 55d0f11c57f..108750ed67d 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -182,6 +182,8 @@ typedef struct MetadataCacheData Oid citusTaskStatusUnscheduledId; Oid citusTaskStatusCancelledId; Oid citusTaskStatusCancellingId; + Oid databaseShardRelationId; + Oid databaseShardPKeyIndexId; Oid distRebalanceStrategyRelationId; Oid distNodeRelationId; Oid distNodeNodeIdIndexId; @@ -2769,12 +2771,34 @@ DistRebalanceStrategyRelationId(void) return MetadataCache.distRebalanceStrategyRelationId; } +/* return oid of citus_catalog.database_sharding relation */ +Oid +DatabaseShardRelationId(void) +{ + CachedRelationNamespaceLookup("database_shard", CitusCatalogNamespaceId(), + &MetadataCache.databaseShardRelationId); + + return MetadataCache.databaseShardRelationId; +} + + +/* return oid of citus_catalog.database_sharding primary key */ +Oid +DatabaseShardPrimaryKeyIndexId(void) +{ + CachedRelationNamespaceLookup("database_shard_pkey", CitusCatalogNamespaceId(), + &MetadataCache.databaseShardPKeyIndexId); + + return MetadataCache.databaseShardPKeyIndexId; +} + + /* return the oid of citus namespace */ Oid CitusCatalogNamespaceId(void) { - CachedNamespaceLookup("citus", &MetadataCache.citusCatalogNamespaceId); + CachedNamespaceLookup("citus_catalog", &MetadataCache.citusCatalogNamespaceId); return MetadataCache.citusCatalogNamespaceId; } @@ -2805,12 +2829,14 @@ DistObjectRelationId(void) true); if (!OidIsValid(MetadataCache.distObjectRelationId)) { + Oid citusNamespaceId = get_namespace_oid("citus", false); + /* * We can only ever reach here while we are creating/altering our extension before * the table is moved to pg_catalog. */ CachedRelationNamespaceLookupExtended("pg_dist_object", - CitusCatalogNamespaceId(), + citusNamespaceId, &MetadataCache.distObjectRelationId, false); } diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index e5d593295a3..3cdd3dbaa1d 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -172,6 +172,8 @@ static GucStringAssignHook OldApplicationNameAssignHook = NULL; */ static bool FinishedStartupCitusBackend = false; +char *CitusMainDatabase = "postgres"; + static object_access_hook_type PrevObjectAccessHook = NULL; #if PG_VERSION_NUM >= PG_VERSION_15 diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index f02f83fe315..9ae57b49aa1 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -40,6 +40,7 @@ typedef enum extern PropSetCmdBehavior PropagateSetCommands; extern bool EnableDDLPropagation; extern int CreateObjectPropagationMode; +extern bool EnableCreateDatabasePropagation; extern bool EnableCreateTypePropagation; extern bool EnableCreateRolePropagation; extern bool EnableAlterRolePropagation; diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index 34b95b859e4..a5e9081a445 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -247,6 +247,7 @@ extern Oid DistLocalGroupIdRelationId(void); extern Oid DistObjectRelationId(void); extern Oid DistEnabledCustomAggregatesId(void); extern Oid DistTenantSchemaRelationId(void); +extern Oid DatabaseShardRelationId(void); /* index oids */ extern Oid DistNodeNodeIdIndexId(void); @@ -271,6 +272,7 @@ extern Oid DistObjectPrimaryKeyIndexId(void); extern Oid DistCleanupPrimaryKeyIndexId(void); extern Oid DistTenantSchemaPrimaryKeyIndexId(void); extern Oid DistTenantSchemaUniqueColocationIdIndexId(void); +extern Oid DatabaseShardPrimaryKeyIndexId(void); /* sequence oids */ extern Oid DistBackgroundJobJobIdSequenceId(void); diff --git a/src/include/distributed/shared_library_init.h b/src/include/distributed/shared_library_init.h index 3764b52fd12..82910d453d6 100644 --- a/src/include/distributed/shared_library_init.h +++ b/src/include/distributed/shared_library_init.h @@ -17,6 +17,8 @@ #define MAX_SHARD_COUNT 64000 #define MAX_SHARD_REPLICATION_FACTOR 100 +extern char *CitusMainDatabase; + extern PGDLLEXPORT ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM; extern PGDLLEXPORT CompressionTypeStr_type extern_CompressionTypeStr; extern PGDLLEXPORT IsColumnarTableAmTable_type extern_IsColumnarTableAmTable; From ca36833b5c3d519b21788a3065aa82fc95aac64b Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 23 Sep 2023 13:49:50 +0300 Subject: [PATCH 004/180] Fix for Pg16 compile error --- src/backend/distributed/database/database_sharding.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/distributed/database/database_sharding.c b/src/backend/distributed/database/database_sharding.c index 51cd4a8f6f1..e8c35c04b21 100644 --- a/src/backend/distributed/database/database_sharding.c +++ b/src/backend/distributed/database/database_sharding.c @@ -35,6 +35,7 @@ #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/fmgroids.h" +#include "catalog/pg_database.h" static void ExecuteCommandInControlDatabase(char *command); From eb1f09338179985326349dbe95368c47a424d98d Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 12:57:05 +0300 Subject: [PATCH 005/180] Fixes runtime and compile errors --- src/backend/distributed/commands/database.c | 229 ++++---- .../distributed/commands/utility_hook.c | 2 +- .../distributed/database/database_lock.c | 53 -- .../distributed/database/database_sharding.c | 546 ------------------ .../distributed/database/database_size.c | 180 ------ .../distributed/metadata/metadata_cache.c | 64 -- src/backend/distributed/shared_library_init.c | 11 + .../distributed/pooler/pgbouncer_manager.h | 47 -- 8 files changed, 125 insertions(+), 1007 deletions(-) delete mode 100644 src/backend/distributed/database/database_lock.c delete mode 100644 src/backend/distributed/database/database_sharding.c delete mode 100644 src/backend/distributed/database/database_size.c delete mode 100644 src/include/distributed/pooler/pgbouncer_manager.h diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 4886ed62e8b..61d89db44ff 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -32,44 +32,43 @@ #include "distributed/deparser.h" #include "distributed/worker_protocol.h" #include "distributed/metadata/distobject.h" -#include "distributed/database/database_sharding.h" #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/adaptive_executor.h" - - /* macros to add DefElems to a list */ -#define DEFELEM_ADD_STRING(options, key, value) { \ - DefElem *elem = makeDefElem(key, (Node *) makeString(value), -1); \ - options = lappend(options, elem); \ -} +#define DEFELEM_ADD_STRING(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *)makeString(value), -1); \ + options = lappend(options, elem); \ + } -#define DEFELEM_ADD_BOOL(options, key, value) { \ - DefElem *elem = makeDefElem(key, (Node *) makeBoolean(value), -1); \ - options = lappend(options, elem); \ -} +#define DEFELEM_ADD_BOOL(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *)makeBoolean(value), -1); \ + options = lappend(options, elem); \ + } -#define DEFELEM_ADD_INT(options, key, value) { \ - DefElem *elem = makeDefElem(key, (Node *) makeInteger(value), -1); \ - options = lappend(options, elem); \ -} +#define DEFELEM_ADD_INT(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *)makeInteger(value), -1); \ + options = lappend(options, elem); \ + } -static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); +static AlterOwnerStmt *RecreateAlterDatabaseOwnerStmt(Oid databaseOid); -static List * CreateDDLTaskList(char *command, List *workerNodeList, - bool outsideTransaction); +static List *CreateDDLTaskList(char *command, List *workerNodeList, + bool outsideTransaction); PG_FUNCTION_INFO_V1(citus_internal_database_command); static Oid get_database_owner(Oid db_oid); -List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +List *PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); /* controlled via GUC */ bool EnableCreateDatabasePropagation = true; bool EnableAlterDatabaseOwner = true; - /* * AlterDatabaseOwnerObjectAddress returns the ObjectAddress of the database that is the * object of the AlterOwnerStmt. Errors if missing_ok is false. @@ -80,14 +79,13 @@ AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_DATABASE); - Oid databaseOid = get_database_oid(strVal((String *) stmt->object), missing_ok); + Oid databaseOid = get_database_oid(strVal((String *)stmt->object), missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, DatabaseRelationId, databaseOid); return list_make1(address); } - /* * DatabaseOwnerDDLCommands returns a list of sql statements to idempotently apply a * change of the database owner on the workers so that the database is owned by the same @@ -96,11 +94,10 @@ AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) List * DatabaseOwnerDDLCommands(const ObjectAddress *address) { - Node *stmt = (Node *) RecreateAlterDatabaseOwnerStmt(address->objectId); + Node *stmt = (Node *)RecreateAlterDatabaseOwnerStmt(address->objectId); return list_make1(DeparseTreeNode(stmt)); } - /* * RecreateAlterDatabaseOwnerStmt creates an AlterOwnerStmt that represents the operation * of changing the owner of the database to its current owner. @@ -111,7 +108,7 @@ RecreateAlterDatabaseOwnerStmt(Oid databaseOid) AlterOwnerStmt *stmt = makeNode(AlterOwnerStmt); stmt->objectType = OBJECT_DATABASE; - stmt->object = (Node *) makeString(get_database_name(databaseOid)); + stmt->object = (Node *)makeString(get_database_name(databaseOid)); Oid ownerOid = get_database_owner(databaseOid); stmt->newowner = makeNode(RoleSpec); @@ -121,7 +118,6 @@ RecreateAlterDatabaseOwnerStmt(Oid databaseOid) return stmt; } - /* * get_database_owner returns the Oid of the role owning the database */ @@ -135,14 +131,13 @@ get_database_owner(Oid db_oid) errmsg("database with OID %u does not exist", db_oid))); } - Oid dba = ((Form_pg_database) GETSTRUCT(tuple))->datdba; + Oid dba = ((Form_pg_database)GETSTRUCT(tuple))->datdba; ReleaseSysCache(tuple); return dba; } - /* * PreprocessGrantOnDatabaseStmt is executed before the statement is applied to the local * postgres instance. @@ -171,88 +166,15 @@ PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *) stmt); + char *sql = DeparseTreeNode((Node *)stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *) sql, + (void *)sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } -/* - * citus_internal_database_command is an internal UDF to - * create/drop a database in an idempotent maner without - * transaction block restrictions. - */ -Datum -citus_internal_database_command(PG_FUNCTION_ARGS) -{ - text *commandText = PG_GETARG_TEXT_P(0); - char *command = text_to_cstring(commandText); - Node *parseTree = ParseTreeNode(command); - - set_config_option("citus.enable_ddl_propagation", "off", - (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, - GUC_ACTION_LOCAL, true, 0, false); - - set_config_option("citus.enable_create_database_propagation", "off", - (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, - GUC_ACTION_LOCAL, true, 0, false); - - if (IsA(parseTree, CreatedbStmt)) - { - CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); - - bool missingOk = true; - Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - if (!OidIsValid(databaseOid)) - { - createdb(NULL, (CreatedbStmt *) parseTree); - } - else - { - /* TODO: check database properties */ - } - } - else if (IsA(parseTree, DropdbStmt)) - { - DropdbStmt *stmt = castNode(DropdbStmt, parseTree); - - bool missingOk = true; - Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - if (!OidIsValid(databaseOid)) - { - /* already dropped? */ - } - else - { - /* remove database from pg_dist_object */ - ObjectAddress dbAddress = { 0 }; - ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); - - if (IsObjectDistributed(&dbAddress)) - { - UnmarkObjectDistributed(&dbAddress); - } - - /* remove database from database shards */ - DeleteDatabaseShardByDatabaseIdLocally(databaseOid); - - DropDatabase(NULL, (DropdbStmt *) parseTree); - } - } - else - { - ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); - } - - PG_RETURN_VOID(); -} - - /* * PreprocessAlterDatabaseStmt is executed before the statement is applied to the local * postgres instance. @@ -273,16 +195,15 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *) stmt); + char *sql = DeparseTreeNode((Node *)stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *) sql, + (void *)sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } - #if PG_VERSION_NUM >= PG_VERSION_15 /* @@ -305,16 +226,15 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *) stmt); + char *sql = DeparseTreeNode((Node *)stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *) sql, + (void *)sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } - #endif /* @@ -380,7 +300,7 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, /* * PostprocessCreatedbStmt creates the plan to synchronize CREATE DATABASE * across nodes. We use the cannotBeExecutedInTransction option to avoid - * u* sending transaction blocks. + * sending transaction blocks. */ List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) @@ -425,24 +345,102 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) } /* synchronize pg_dist_object records */ - ObjectAddress dbAddress = { 0 }; + ObjectAddress dbAddress = {0}; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); MarkObjectDistributed(&dbAddress); + return NIL; } +/* + * citus_internal_database_command is an internal UDF to + * create/drop a database in an idempotent maner without + * transaction block restrictions. + */ +Datum citus_internal_database_command(PG_FUNCTION_ARGS) +{ + int saveNestLevel = NewGUCNestLevel(); + text *commandText = PG_GETARG_TEXT_P(0); + char *command = text_to_cstring(commandText); + Node *parseTree = ParseTreeNode(command); + + ereport(NOTICE, (errmsg("test internal pre"), + errhint("test pre hint"))); + + set_config_option("citus.enable_ddl_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + set_config_option("citus.enable_create_database_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + if (IsA(parseTree, CreatedbStmt)) + { + CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); + + bool missingOk = true; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + if (!OidIsValid(databaseOid)) + { + createdb(NULL, (CreatedbStmt *)parseTree); + } + else + { + /* TODO: check database properties */ + } + } + else if (IsA(parseTree, DropdbStmt)) + { + DropdbStmt *stmt = castNode(DropdbStmt, parseTree); + + bool missingOk = true; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + if (!OidIsValid(databaseOid)) + { + /* already dropped? */ + } + else + { + /* remove database from pg_dist_object */ + ObjectAddress dbAddress = {0}; + ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); + + if (IsObjectDistributed(&dbAddress)) + { + UnmarkObjectDistributed(&dbAddress); + } + + // /* remove database from database shards */ + // DeleteDatabaseShardByDatabaseIdLocally(databaseOid); + + DropDatabase(NULL, (DropdbStmt *)parseTree); + } + } + else + { + ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); + } + + AtEOXact_GUC(true, saveNestLevel); + + PG_RETURN_VOID(); +} + List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext) + ProcessUtilityContext processUtilityContext) { if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } - DropdbStmt *stmt = (DropdbStmt *) node; + DropdbStmt *stmt = (DropdbStmt *)node; char *databaseName = stmt->dbname; bool missingOk = true; Oid databaseOid = get_database_oid(databaseName, missingOk); @@ -452,7 +450,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } - ObjectAddress dbAddress = { 0 }; + ObjectAddress dbAddress = {0}; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); if (!IsObjectDistributed(&dbAddress)) { @@ -472,7 +470,6 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, "SELECT pg_catalog.citus_internal_database_command(%s)", quote_literal_cstr(dropDatabaseCommand)); - /* we execute here to avoid EnsureCoordinator check in ExecuteDistributedDDLJob */ bool outsideTransaction = false; List *taskList = CreateDDLTaskList(internalDropCommand->data, workerNodes, diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index cf8e0644eec..a818a1ad790 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -694,7 +694,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } /* inform the user about potential caveats */ - if (IsA(parsetree, CreatedbStmt)) + if (IsA(parsetree, CreatedbStmt) &&!EnableCreateDatabasePropagation) { if (EnableUnsupportedFeatureMessages) { diff --git a/src/backend/distributed/database/database_lock.c b/src/backend/distributed/database/database_lock.c deleted file mode 100644 index 64757832fc2..00000000000 --- a/src/backend/distributed/database/database_lock.c +++ /dev/null @@ -1,53 +0,0 @@ -/*------------------------------------------------------------------------- - * - * database_lock.c - * Functions for locking a database. - * - * Copyright (c) Microsoft, Inc. - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" -#include "funcapi.h" -#include "fmgr.h" -#include "miscadmin.h" - -#include "catalog/pg_database.h" -#include "commands/dbcommands.h" -#include "distributed/metadata_cache.h" -#include "storage/lmgr.h" - - -static void CitusDatabaseLock(Oid databaseId); - - -PG_FUNCTION_INFO_V1(citus_database_lock_by_name); - - -/* - * citus_database_lock locks the given database in access exclusive mode - * to temporarily block new connections. - */ -Datum -citus_database_lock_by_name(PG_FUNCTION_ARGS) -{ - CheckCitusVersion(ERROR); - - Name databaseName = PG_GETARG_NAME(0); - bool missingOk = false; - Oid databaseId = get_database_oid(NameStr(*databaseName), missingOk); - - CitusDatabaseLock(databaseId); - - PG_RETURN_VOID(); -} - - -/* - * CitusDatabaseLock locks a database for new connections. - */ -static void -CitusDatabaseLock(Oid databaseId) -{ - LockSharedObject(DatabaseRelationId, databaseId, 0, ExclusiveLock); -} diff --git a/src/backend/distributed/database/database_sharding.c b/src/backend/distributed/database/database_sharding.c deleted file mode 100644 index e8c35c04b21..00000000000 --- a/src/backend/distributed/database/database_sharding.c +++ /dev/null @@ -1,546 +0,0 @@ -/*------------------------------------------------------------------------- - * - * database_sharding.c - * - * This file contains module-level definitions. - * - * Copyright (c) 2023, Microsoft, Inc. - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" -#include "fmgr.h" -#include "miscadmin.h" - -#include "citus_version.h" -#include "pg_version_compat.h" - -#include "access/genam.h" -#include "commands/dbcommands.h" -#include "distributed/connection_management.h" -#include "distributed/database/database_sharding.h" -#include "distributed/deparser.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/listutils.h" -#include "distributed/metadata_sync.h" -#include "distributed/pooler/pgbouncer_manager.h" -#include "distributed/remote_commands.h" -#include "distributed/shared_library_init.h" -#include "distributed/worker_transaction.h" -#include "executor/spi.h" -#include "nodes/makefuncs.h" -#include "nodes/parsenodes.h" -#include "postmaster/postmaster.h" -#include "tcop/utility.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "catalog/pg_database.h" - - -static void ExecuteCommandInControlDatabase(char *command); -static void AllowConnectionsOnlyOnNodeGroup(Oid databaseOid, Oid nodeGroupId); -static void InsertDatabaseShardAssignment(Oid databaseOid, int nodeGroupId); -static void InsertDatabaseShardAssignmentLocally(Oid databaseOid, int nodeGroupId); -static void InsertDatabaseShardAssignmentOnOtherNodes(Oid databaseOid, int nodeGroupId); -static void DeleteDatabaseShardByDatabaseId(Oid databaseOid); -static void DeleteDatabaseShardByDatabaseIdOnOtherNodes(Oid databaseOid); -static DatabaseShard * TupleToDatabaseShard(HeapTuple heapTuple, - TupleDesc tupleDescriptor); -static char * DeleteDatabaseShardByDatabaseIdCommand(Oid databaseOid); - - -PG_FUNCTION_INFO_V1(database_shard_assign); -PG_FUNCTION_INFO_V1(citus_internal_add_database_shard); -PG_FUNCTION_INFO_V1(citus_internal_delete_database_shard); - - -/* citus.enable_database_sharding setting */ -bool EnableDatabaseSharding = false; - -/* citus.database_sharding_pgbouncer_file setting */ -char *DatabaseShardingPgBouncerFile = ""; - - -/* - * PreProcessUtilityInDatabaseShard handles DDL commands that occur within a - * database shard and require global coordination: - * - CREATE/ALTER/DROP DATABASE - * - CREATE/ALTER/DROP ROLE/USER/GROUP - */ -void -PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, - ProcessUtilityContext context, - bool *runPreviousUtilityHook) -{ - if (!EnableDatabaseSharding || context != PROCESS_UTILITY_TOPLEVEL) - { - return; - } - - if (EnableCreateDatabasePropagation) - { - if (IsA(parseTree, CreatedbStmt)) - { - char *command = DeparseCreateDatabaseStmt(parseTree); - ExecuteCommandInControlDatabase(command); - - /* command is fully delegated to control database */ - *runPreviousUtilityHook = false; - } - else if (IsA(parseTree, DropdbStmt)) - { - char *command = DeparseDropDatabaseStmt(parseTree); - ExecuteCommandInControlDatabase(command); - - /* command is fully delegated to control database */ - *runPreviousUtilityHook = false; - } - } -} - - -/* - * PostProcessUtilityInDatabaseShard is currently a noop. - */ -void -PostProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, - ProcessUtilityContext context) -{ - if (!EnableDatabaseSharding || context != PROCESS_UTILITY_TOPLEVEL) - { - return; - } -} - - -/* - * ExecuteCommandInControlDatabase connects to localhost to execute a command - * in the main Citus database. - */ -static void -ExecuteCommandInControlDatabase(char *command) -{ - int connectionFlag = FORCE_NEW_CONNECTION; - - MultiConnection *connection = - GetNodeUserDatabaseConnection(connectionFlag, LocalHostName, PostPortNumber, - NULL, CitusMainDatabase); - - ExecuteCriticalRemoteCommand(connection, - "SET application_name TO 'citus_database_shard'"); - ExecuteCriticalRemoteCommand(connection, command); - CloseConnection(connection); -} - - -/* - * database_shard_assign assigns an existing database to a node. - */ -Datum -database_shard_assign(PG_FUNCTION_ARGS) -{ - CheckCitusVersion(ERROR); - - char *databaseName = text_to_cstring(PG_GETARG_TEXT_P(0)); - - bool missingOk = false; - Oid databaseOid = get_database_oid(databaseName, missingOk); - - if (!pg_database_ownercheck(databaseOid, GetUserId())) - { - ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("permission denied to assign database \"%s\" " - "to a shard", - databaseName))); - } - - if (GetDatabaseShardByOid(databaseOid) != NULL) - { - ereport(ERROR, (errmsg("database is already assigned to a shard"))); - } - - AssignDatabaseToShard(databaseOid); - - PG_RETURN_VOID(); -} - - -/* - * AssignDatabaseToShard finds a suitable node for the given - * database and assigns it. - */ -void -AssignDatabaseToShard(Oid databaseOid) -{ - int nodeGroupId = GetLocalGroupId(); - - List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); - if (list_length(workerNodes) > 0) - { - /* TODO: actually look for available space */ - int workerNodeIndex = databaseOid % list_length(workerNodes); - WorkerNode *workerNode = list_nth(workerNodes, workerNodeIndex); - nodeGroupId = workerNode->groupId; - } - - InsertDatabaseShardAssignment(databaseOid, nodeGroupId); - AllowConnectionsOnlyOnNodeGroup(databaseOid, nodeGroupId); - - ReconfigurePgBouncersOnCommit = true; -} - - -/* - * AllowConnectionsOnlyOnNodeGroup sets the ALLOW_CONNECTIONS properties on - * the database to false, except on nodeGroupId. - */ -static void -AllowConnectionsOnlyOnNodeGroup(Oid databaseOid, Oid nodeGroupId) -{ - StringInfo command = makeStringInfo(); - char *databaseName = get_database_name(databaseOid); - - List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); - WorkerNode *workerNode = NULL; - - foreach_ptr(workerNode, workerNodes) - { - resetStringInfo(command); - - if (workerNode->groupId == nodeGroupId) - { - appendStringInfo(command, "GRANT CONNECT ON DATABASE %s TO public", - quote_identifier(databaseName)); - } - else - { - appendStringInfo(command, "REVOKE CONNECT ON DATABASE %s FROM public", - quote_identifier(databaseName)); - } - - if (workerNode->groupId == GetLocalGroupId()) - { - ExecuteQueryViaSPI(command->data, SPI_OK_UTILITY); - } - else - { - SendCommandToWorker(workerNode->workerName, workerNode->workerPort, - command->data); - } - } -} - - -/* - * InsertDatabaseShardAssignment inserts a record into the local - * citus_catalog.database_sharding table. - */ -static void -InsertDatabaseShardAssignment(Oid databaseOid, int nodeGroupId) -{ - InsertDatabaseShardAssignmentLocally(databaseOid, nodeGroupId); - - if (EnableMetadataSync) - { - InsertDatabaseShardAssignmentOnOtherNodes(databaseOid, nodeGroupId); - } -} - - -/* - * InsertDatabaseShardAssignmentLocally inserts a record into the local - * citus_catalog.database_sharding table. - */ -static void -InsertDatabaseShardAssignmentLocally(Oid databaseOid, int nodeGroupId) -{ - Datum values[Natts_database_shard]; - bool isNulls[Natts_database_shard]; - - /* form new shard tuple */ - memset(values, 0, sizeof(values)); - memset(isNulls, false, sizeof(isNulls)); - - values[Anum_database_shard_database_id - 1] = ObjectIdGetDatum(databaseOid); - values[Anum_database_shard_node_group_id - 1] = Int32GetDatum(nodeGroupId); - values[Anum_database_shard_is_available - 1] = BoolGetDatum(true); - - /* open shard relation and insert new tuple */ - Relation databaseShardTable = table_open(DatabaseShardRelationId(), RowExclusiveLock); - - TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); - HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); - - CatalogTupleInsert(databaseShardTable, heapTuple); - - CommandCounterIncrement(); - table_close(databaseShardTable, NoLock); -} - - -/* - * InsertDatabaseShardAssignmentOnOtherNodes inserts a record into the - * citus_catalog.database_sharding table on other nodes. - */ -static void -InsertDatabaseShardAssignmentOnOtherNodes(Oid databaseOid, int nodeGroupId) -{ - char *insertCommand = InsertDatabaseShardAssignmentCommand(databaseOid, nodeGroupId); - SendCommandToWorkersWithMetadata(insertCommand); -} - - -/* - * UpdateDatabaseShard updates a database shard after it is moved to a new node. - */ -void -UpdateDatabaseShard(Oid databaseOid, int targetNodeGroupId) -{ - DeleteDatabaseShardByDatabaseId(databaseOid); - InsertDatabaseShardAssignment(databaseOid, targetNodeGroupId); - AllowConnectionsOnlyOnNodeGroup(databaseOid, targetNodeGroupId); -} - - -/* - * DeleteDatabaseShardByDatabaseId deletes a record from the - * citus_catalog.database_sharding table. - */ -static void -DeleteDatabaseShardByDatabaseId(Oid databaseOid) -{ - DeleteDatabaseShardByDatabaseIdLocally(databaseOid); - - if (EnableMetadataSync) - { - DeleteDatabaseShardByDatabaseIdOnOtherNodes(databaseOid); - } -} - - -/* - * DeleteDatabaseShardByDatabaseIdLocally deletes a database_shard record by database OID. - */ -void -DeleteDatabaseShardByDatabaseIdLocally(Oid databaseOid) -{ - Relation databaseShardTable = table_open(DatabaseShardRelationId(), - RowExclusiveLock); - - const int scanKeyCount = 1; - ScanKeyData scanKey[1]; - bool indexOK = true; - - ScanKeyInit(&scanKey[0], Anum_database_shard_database_id, - BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(databaseOid)); - - SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, - DatabaseShardPrimaryKeyIndexId(), - indexOK, - NULL, scanKeyCount, scanKey); - - HeapTuple heapTuple = systable_getnext(scanDescriptor); - if (heapTuple != NULL) - { - simple_heap_delete(databaseShardTable, &heapTuple->t_self); - } - - systable_endscan(scanDescriptor); - - CommandCounterIncrement(); - table_close(databaseShardTable, NoLock); -} - - -/* - * DeleteDatabaseShardByDatabaseIdOnOtherNodes deletes a record from the - * citus_catalog.database_sharding table on other nodes. - */ -static void -DeleteDatabaseShardByDatabaseIdOnOtherNodes(Oid databaseOid) -{ - char *deleteCommand = DeleteDatabaseShardByDatabaseIdCommand(databaseOid); - SendCommandToWorkersWithMetadata(deleteCommand); -} - - -/* - * ListDatabaseShards lists all database shards in citus_catalog.database_shard. - */ -List * -ListDatabaseShards(void) -{ - Relation databaseShardTable = table_open(DatabaseShardRelationId(), AccessShareLock); - TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); - - List *dbShardList = NIL; - int scanKeyCount = 0; - bool indexOK = false; - - SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, InvalidOid, - indexOK, NULL, scanKeyCount, NULL); - - HeapTuple heapTuple = NULL; - while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) - { - DatabaseShard *dbShard = TupleToDatabaseShard(heapTuple, tupleDescriptor); - dbShardList = lappend(dbShardList, dbShard); - } - - systable_endscan(scanDescriptor); - table_close(databaseShardTable, NoLock); - - return dbShardList; -} - - -/* - * GetDatabaseShardByOid gets a database shard by database OID or - * NULL if no database shard could be found. - */ -DatabaseShard * -GetDatabaseShardByOid(Oid databaseOid) -{ - DatabaseShard *result = NULL; - - Relation databaseShardTable = table_open(DatabaseShardRelationId(), AccessShareLock); - TupleDesc tupleDescriptor = RelationGetDescr(databaseShardTable); - - const int scanKeyCount = 1; - ScanKeyData scanKey[1]; - bool indexOK = true; - - ScanKeyInit(&scanKey[0], Anum_database_shard_database_id, - BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(databaseOid)); - - SysScanDesc scanDescriptor = systable_beginscan(databaseShardTable, - DatabaseShardPrimaryKeyIndexId(), - indexOK, - NULL, scanKeyCount, scanKey); - - HeapTuple heapTuple = systable_getnext(scanDescriptor); - if (HeapTupleIsValid(heapTuple)) - { - result = TupleToDatabaseShard(heapTuple, tupleDescriptor); - } - - systable_endscan(scanDescriptor); - table_close(databaseShardTable, NoLock); - - return result; -} - - -/* - * TupleToDatabaseShard converts a database_shard record tuple into a DatabaseShard struct. - */ -static DatabaseShard * -TupleToDatabaseShard(HeapTuple heapTuple, TupleDesc tupleDescriptor) -{ - Datum datumArray[Natts_database_shard]; - bool isNullArray[Natts_database_shard]; - heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); - - DatabaseShard *record = palloc0(sizeof(DatabaseShard)); - - record->databaseOid = - DatumGetObjectId(datumArray[Anum_database_shard_database_id - 1]); - - record->nodeGroupId = - DatumGetInt32(datumArray[Anum_database_shard_node_group_id - 1]); - - record->isAvailable = - DatumGetBool(datumArray[Anum_database_shard_is_available - 1]); - - return record; -} - - -/* - * citus_internal_add_database_shard is an internal UDF to - * add a row to database_shard. - */ -Datum -citus_internal_add_database_shard(PG_FUNCTION_ARGS) -{ - char *databaseName = TextDatumGetCString(PG_GETARG_DATUM(0)); - int nodeGroupId = PG_GETARG_INT32(1); - - bool missingOk = false; - Oid databaseOid = get_database_oid(databaseName, missingOk); - - if (!pg_database_ownercheck(databaseOid, GetUserId())) - { - aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, - databaseName); - } - - InsertDatabaseShardAssignmentLocally(databaseOid, nodeGroupId); - - - PG_RETURN_VOID(); -} - - -/* - * InsertDatabaseShardAssignmentCommand returns a command to insert a database shard - * assignment into the metadata on a remote node. - */ -char * -InsertDatabaseShardAssignmentCommand(Oid databaseOid, int nodeGroupId) -{ - StringInfo command = makeStringInfo(); - char *databaseName = get_database_name(databaseOid); - - appendStringInfo(command, - "SELECT pg_catalog.citus_internal_add_database_shard(%s,%d)", - quote_literal_cstr(databaseName), - nodeGroupId); - - return command->data; -} - - -/* - * citus_internal_delete_database_shard is an internal UDF to - * delete a row from database_shard. - */ -Datum -citus_internal_delete_database_shard(PG_FUNCTION_ARGS) -{ - char *databaseName = TextDatumGetCString(PG_GETARG_DATUM(0)); - - bool missingOk = false; - Oid databaseOid = get_database_oid(databaseName, missingOk); - - if (!pg_database_ownercheck(databaseOid, GetUserId())) - { - aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_DATABASE, - databaseName); - } - - DeleteDatabaseShardByDatabaseIdLocally(databaseOid); - - - - PG_RETURN_VOID(); -} - - -/* - * DeleteDatabaseShardByDatabaseIdCommand returns a command to delete a database shard - * assignment from the metadata on a remote node. - */ -static char * -DeleteDatabaseShardByDatabaseIdCommand(Oid databaseOid) -{ - StringInfo command = makeStringInfo(); - char *databaseName = get_database_name(databaseOid); - - appendStringInfo(command, - "SELECT pg_catalog.citus_internal_delete_database_shard(%s)", - quote_literal_cstr(databaseName)); - - return command->data; -} diff --git a/src/backend/distributed/database/database_size.c b/src/backend/distributed/database/database_size.c deleted file mode 100644 index 46e72ad13b8..00000000000 --- a/src/backend/distributed/database/database_size.c +++ /dev/null @@ -1,180 +0,0 @@ -/*------------------------------------------------------------------------- - * - * database_size.c - * Functions for getting the size of a database. - * - * Copyright (c) Microsoft, Inc. - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" -#include "funcapi.h" -#include "fmgr.h" -#include "miscadmin.h" - -#include "commands/dbcommands.h" -#include "distributed/citus_safe_lib.h" -#include "distributed/database/database_sharding.h" -#include "distributed/listutils.h" -#include "distributed/metadata_cache.h" -#include "distributed/remote_commands.h" -#include "distributed/worker_transaction.h" -#include "utils/builtins.h" - - -static int64 GetLocalDatabaseSize(Oid databaseId); -static int64 CitusDatabaseShardSize(DatabaseShard *dbShard); -static int64 CitusDatabaseSizeOnNodeList(Oid databaseId, List *workerNodeList); - - -PG_FUNCTION_INFO_V1(citus_database_size_oid); -PG_FUNCTION_INFO_V1(citus_database_size_name); - - -/* - * citus_database_size_oid returns the size of a Citus database - * with the given oid. - */ -Datum -citus_database_size_oid(PG_FUNCTION_ARGS) -{ - CheckCitusVersion(ERROR); - - Oid databaseId = PG_GETARG_OID(0); - int64 size = CitusDatabaseSize(databaseId); - - PG_RETURN_INT64(size); -} - - -/* - * citus_database_size_name returns the size of a Citus database - * with the given name. - */ -Datum -citus_database_size_name(PG_FUNCTION_ARGS) -{ - CheckCitusVersion(ERROR); - - Name databaseName = PG_GETARG_NAME(0); - bool missingOk = false; - Oid databaseId = get_database_oid(NameStr(*databaseName), missingOk); - - int64 size = CitusDatabaseSize(databaseId); - - PG_RETURN_INT64(size); -} - - -/* - * CitusDatabaseSize returns the size of a Citus database. - */ -int64 -CitusDatabaseSize(Oid databaseId) -{ - DatabaseShard *dbShard = GetDatabaseShardByOid(databaseId); - if (dbShard != NULL) - { - /* for known database shards, get the remote size */ - return CitusDatabaseShardSize(dbShard); - } - - if (databaseId == MyDatabaseId) - { - /* for the current database, get the size from all nodes */ - List *workerNodes = TargetWorkerSetNodeList(ALL_SHARD_NODES, RowShareLock); - return CitusDatabaseSizeOnNodeList(databaseId, workerNodes); - } - - /* for other databases, get the local size */ - /* TODO: get it from main database? */ - return GetLocalDatabaseSize(databaseId); -} - - -/* - * GetLocalDatabaseSize returns the local database size by calling pg_database_size. - */ -static int64 -GetLocalDatabaseSize(Oid databaseId) -{ - Datum databaseIdDatum = ObjectIdGetDatum(databaseId); - Datum sizeDatum = DirectFunctionCall1(pg_database_size_oid, databaseIdDatum); - return DatumGetInt64(sizeDatum); -} - - -/* - * CitusDatabaseShardSize gets the database size for a specific - * shard. - */ -static int64 -CitusDatabaseShardSize(DatabaseShard *dbShard) -{ - WorkerNode *workerNode = LookupNodeForGroup(dbShard->nodeGroupId); - - return CitusDatabaseSizeOnNodeList(dbShard->databaseOid, list_make1(workerNode)); -} - - -/* - * CitusDatabaseSizeOnNodeList returns the sum of the sizes - * for a given database from all nodes in the list. - */ -static int64 -CitusDatabaseSizeOnNodeList(Oid databaseId, List *workerNodeList) -{ - int64 size = 0; - - bool raiseInterrupts = true; - - char *databaseName = get_database_name(databaseId); - char *command = psprintf("SELECT pg_catalog.pg_database_size(%s)", - quote_literal_cstr(databaseName)); - - WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) - { - if (workerNode->groupId == GetLocalGroupId()) - { - return GetLocalDatabaseSize(databaseId); - } - - int connectionFlags = 0; - MultiConnection *connection = GetNodeConnection(connectionFlags, - workerNode->workerName, - workerNode->workerPort); - - int querySent = SendRemoteCommand(connection, command); - if (querySent == 0) - { - ReportConnectionError(connection, ERROR); - } - - PGresult *result = GetRemoteCommandResult(connection, raiseInterrupts); - if (!IsResponseOK(result)) - { - ReportResultError(connection, result, ERROR); - } - - if (PQntuples(result) != 1 || PQnfields(result) != 1) - { - PQclear(result); - ClearResults(connection, raiseInterrupts); - - ereport(ERROR, (errmsg("unexpected number of columns returned by: %s", - command))); - } - - if (!PQgetisnull(result, 0, 0)) - { - char *sizeString = PQgetvalue(result, 0, 0); - size += SafeStringToUint64(sizeString); - } - - PQclear(result); - ClearResults(connection, raiseInterrupts); - } - - return size; -} diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 108750ed67d..7345b1839eb 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -208,7 +208,6 @@ typedef struct MetadataCacheData Oid distTransactionGroupIndexId; Oid distTenantSchemaPrimaryKeyIndexId; Oid distTenantSchemaUniqueColocationIdIndexId; - Oid citusCatalogNamespaceId; Oid copyFormatTypeId; Oid readIntermediateResultFuncId; Oid readIntermediateResultArrayFuncId; @@ -312,7 +311,6 @@ static HeapTuple LookupDistPartitionTuple(Relation pgDistPartition, Oid relation static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, Oid *columnTypeId, int32 *columnTypeMod, Oid *intervalTypeId, int32 *intervalTypeMod); -static void CachedNamespaceLookup(const char *nspname, Oid *cachedOid); static void CachedRelationLookup(const char *relationName, Oid *cachedOid); static void CachedRelationLookupExtended(const char *relationName, Oid *cachedOid, bool missing_ok); @@ -2771,38 +2769,9 @@ DistRebalanceStrategyRelationId(void) return MetadataCache.distRebalanceStrategyRelationId; } -/* return oid of citus_catalog.database_sharding relation */ -Oid -DatabaseShardRelationId(void) -{ - CachedRelationNamespaceLookup("database_shard", CitusCatalogNamespaceId(), - &MetadataCache.databaseShardRelationId); - - return MetadataCache.databaseShardRelationId; -} - - -/* return oid of citus_catalog.database_sharding primary key */ -Oid -DatabaseShardPrimaryKeyIndexId(void) -{ - CachedRelationNamespaceLookup("database_shard_pkey", CitusCatalogNamespaceId(), - &MetadataCache.databaseShardPKeyIndexId); - - return MetadataCache.databaseShardPKeyIndexId; -} -/* return the oid of citus namespace */ -Oid -CitusCatalogNamespaceId(void) -{ - CachedNamespaceLookup("citus_catalog", &MetadataCache.citusCatalogNamespaceId); - return MetadataCache.citusCatalogNamespaceId; -} - - /* return oid of pg_dist_object relation */ Oid DistObjectRelationId(void) @@ -2870,17 +2839,6 @@ DistObjectPrimaryKeyIndexId(void) &MetadataCache.distObjectPrimaryKeyIndexId, true); - if (!OidIsValid(MetadataCache.distObjectPrimaryKeyIndexId)) - { - /* - * We can only ever reach here while we are creating/altering our extension before - * the table is moved to pg_catalog. - */ - CachedRelationNamespaceLookupExtended("pg_dist_object_pkey", - CitusCatalogNamespaceId(), - &MetadataCache.distObjectPrimaryKeyIndexId, - false); - } return MetadataCache.distObjectPrimaryKeyIndexId; } @@ -5446,28 +5404,6 @@ DeformedDistShardTupleToShardInterval(Datum *datumArray, bool *isNullArray, } -/* - * CachedNamespaceLookup performs a cached lookup for the namespace (schema), with the - * result cached in cachedOid. - */ -static void -CachedNamespaceLookup(const char *nspname, Oid *cachedOid) -{ - /* force callbacks to be registered, so we always get notified upon changes */ - InitializeCaches(); - - if (*cachedOid == InvalidOid) - { - *cachedOid = get_namespace_oid(nspname, true); - - if (*cachedOid == InvalidOid) - { - ereport(ERROR, (errmsg( - "cache lookup failed for namespace %s, called too early?", - nspname))); - } - } -} /* diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 3cdd3dbaa1d..c510c2d3019 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1264,6 +1264,17 @@ RegisterCitusConfigVariables(void) GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.enable_create_database_propagation", + gettext_noop("Enables propagating CREATE DATABASE " + "and DROP DATABASE statements to workers"), + NULL, + &EnableCreateDatabasePropagation, + true, + PGC_USERSET, + GUC_STANDARD, + NULL, NULL, NULL); + DefineCustomBoolVariable( "citus.enable_create_role_propagation", gettext_noop("Enables propagating CREATE ROLE " diff --git a/src/include/distributed/pooler/pgbouncer_manager.h b/src/include/distributed/pooler/pgbouncer_manager.h deleted file mode 100644 index 530d1b1ee39..00000000000 --- a/src/include/distributed/pooler/pgbouncer_manager.h +++ /dev/null @@ -1,47 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pgbouncer_manager.h - * Functions for managing outbound pgbouncers - * - * Copyright (c) Citus Data, Inc. - * - *------------------------------------------------------------------------- - */ - -#ifndef PGBOUNCER_MANAGER_H -#define PGBOUNCER_MANAGER_H - -/* default number of inbound pgbouncer processes (0 is disabled) */ -#define PGBOUNCER_INBOUND_PROCS_DEFAULT 0 - -/* bits reserved for local pgbouncer ID in the pgbouncer peer_id */ -#define PGBOUNCER_PEER_ID_LOCAL_ID_BITS 5 - -/* maximum number of inbound pgbouncer processes */ -#define PGBOUNCER_INBOUND_PROCS_MAX ((1 << PGBOUNCER_PEER_ID_LOCAL_ID_BITS) - 1) - - -/* GUC variable that sets the number of inbound pgbouncer procs */ -extern int PgBouncerInboundProcs; - -/* GUC variable that sets the inbound pgbouncer port */ -extern int PgBouncerInboundPort; - -/* GUC variable that sets the path to pgbouncer executable */ -extern char *PgBouncerPath; - -/* GUC variable that sets a pgbouncer file to include */ -extern char *PgBouncerIncludeConfig; - -/* global variable to request pgbouncer reconfiguration */ -extern bool ReconfigurePgBouncersOnCommit; - -void InitializeSharedPgBouncerManager(void); -size_t SharedPgBouncerManagerShmemSize(void); -void PgBouncerManagerMain(Datum arg); -bool PauseDatabaseOnInboundPgBouncers(char *databaseName); -bool ResumeDatabaseOnInboundPgBouncers(char *databaseName); -void TriggerPgBouncerReconfigureIfNeeded(void); - - -#endif /* PGBOUNCER_MANAGER_H */ From 40893e6ffc2df399a4bee3f15d80a518f75a459e Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 13:17:49 +0300 Subject: [PATCH 006/180] Removes unnecessary files --- .../distributed/metadata/metadata_cache.c | 2 - .../distributed/database/database_sharding.h | 53 ------------------- src/include/distributed/metadata_cache.h | 2 - src/include/distributed/shared_library_init.h | 2 - 4 files changed, 59 deletions(-) delete mode 100644 src/include/distributed/database/database_sharding.h diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 7345b1839eb..531db2796f6 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -182,8 +182,6 @@ typedef struct MetadataCacheData Oid citusTaskStatusUnscheduledId; Oid citusTaskStatusCancelledId; Oid citusTaskStatusCancellingId; - Oid databaseShardRelationId; - Oid databaseShardPKeyIndexId; Oid distRebalanceStrategyRelationId; Oid distNodeRelationId; Oid distNodeNodeIdIndexId; diff --git a/src/include/distributed/database/database_sharding.h b/src/include/distributed/database/database_sharding.h deleted file mode 100644 index 390be1383b0..00000000000 --- a/src/include/distributed/database/database_sharding.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * database_sharding.h - * - * Data structure definition for managing backend data and related function - * - * Copyright (c) Microsoft, Inc. - */ - -#ifndef DATABASE_SHARDING_H -#define DATABASE_SHARDING_H - - -#include "tcop/utility.h" - - -/* attributes of citus_catalog.database_shard */ -#define Natts_database_shard 3 -#define Anum_database_shard_database_id 1 -#define Anum_database_shard_node_group_id 2 -#define Anum_database_shard_is_available 3 - - -typedef struct DatabaseShard -{ - /* database oid */ - Oid databaseOid; - - /* node group on which the database shard is placed */ - int nodeGroupId; - - /* whether the database shard is available */ - bool isAvailable; -} DatabaseShard; - -/* citus.enable_database_sharding setting */ -extern bool EnableDatabaseSharding; - -void PreProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, - ProcessUtilityContext context, - bool *runPreviousUtilityHook); -void PostProcessUtilityInDatabaseShard(Node *parseTree, const char *queryString, - ProcessUtilityContext context); -bool DatabaseShardingEnabled(void); -void AssignDatabaseToShard(Oid databaseOid); -void UpdateDatabaseShard(Oid databaseOid, int targetNodeGroupId); -void DeleteDatabaseShardByDatabaseIdLocally(Oid databaseOid); -DatabaseShard * GetDatabaseShardByOid(Oid databaseOid); -List * ListDatabaseShards(void); -int64 CitusDatabaseSize(Oid databaseId); -char * InsertDatabaseShardAssignmentCommand(Oid databaseOid, int nodeGroupId); - - -#endif diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index a5e9081a445..34b95b859e4 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -247,7 +247,6 @@ extern Oid DistLocalGroupIdRelationId(void); extern Oid DistObjectRelationId(void); extern Oid DistEnabledCustomAggregatesId(void); extern Oid DistTenantSchemaRelationId(void); -extern Oid DatabaseShardRelationId(void); /* index oids */ extern Oid DistNodeNodeIdIndexId(void); @@ -272,7 +271,6 @@ extern Oid DistObjectPrimaryKeyIndexId(void); extern Oid DistCleanupPrimaryKeyIndexId(void); extern Oid DistTenantSchemaPrimaryKeyIndexId(void); extern Oid DistTenantSchemaUniqueColocationIdIndexId(void); -extern Oid DatabaseShardPrimaryKeyIndexId(void); /* sequence oids */ extern Oid DistBackgroundJobJobIdSequenceId(void); diff --git a/src/include/distributed/shared_library_init.h b/src/include/distributed/shared_library_init.h index 82910d453d6..3764b52fd12 100644 --- a/src/include/distributed/shared_library_init.h +++ b/src/include/distributed/shared_library_init.h @@ -17,8 +17,6 @@ #define MAX_SHARD_COUNT 64000 #define MAX_SHARD_REPLICATION_FACTOR 100 -extern char *CitusMainDatabase; - extern PGDLLEXPORT ColumnarSupportsIndexAM_type extern_ColumnarSupportsIndexAM; extern PGDLLEXPORT CompressionTypeStr_type extern_CompressionTypeStr; extern PGDLLEXPORT IsColumnarTableAmTable_type extern_IsColumnarTableAmTable; From 1e7c18cb619bd8cd39ff985c02d108e29ce49777 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 13:19:36 +0300 Subject: [PATCH 007/180] Removes unnecessary code --- src/backend/distributed/metadata/metadata_cache.c | 1 + src/backend/distributed/shared_library_init.c | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 531db2796f6..862eb0bfc8e 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -206,6 +206,7 @@ typedef struct MetadataCacheData Oid distTransactionGroupIndexId; Oid distTenantSchemaPrimaryKeyIndexId; Oid distTenantSchemaUniqueColocationIdIndexId; + Oid citusCatalogNamespaceId; Oid copyFormatTypeId; Oid readIntermediateResultFuncId; Oid readIntermediateResultArrayFuncId; diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index c510c2d3019..886517464e5 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -172,8 +172,6 @@ static GucStringAssignHook OldApplicationNameAssignHook = NULL; */ static bool FinishedStartupCitusBackend = false; -char *CitusMainDatabase = "postgres"; - static object_access_hook_type PrevObjectAccessHook = NULL; #if PG_VERSION_NUM >= PG_VERSION_15 From cef9ac702e7fe1f57d68ede4d208f7919205f8ae Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 13:32:39 +0300 Subject: [PATCH 008/180] Fixes indentation --- src/backend/distributed/commands/database.c | 91 ++- .../commands/distribute_object_ops.c | 1 + .../distributed/commands/utility_hook.c | 2 +- .../deparser/deparse_database_stmts.c | 37 +- .../distributed/metadata/metadata_cache.c | 4 - src/include/distributed/commands.h | 757 ++++++++++-------- src/include/distributed/deparser.h | 1 - 7 files changed, 479 insertions(+), 414 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 61d89db44ff..c760aaa705b 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -37,33 +37,33 @@ #include "distributed/adaptive_executor.h" /* macros to add DefElems to a list */ -#define DEFELEM_ADD_STRING(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *)makeString(value), -1); \ - options = lappend(options, elem); \ +#define DEFELEM_ADD_STRING(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *) makeString(value), -1); \ + options = lappend(options, elem); \ } -#define DEFELEM_ADD_BOOL(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *)makeBoolean(value), -1); \ - options = lappend(options, elem); \ +#define DEFELEM_ADD_BOOL(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *) makeBoolean(value), -1); \ + options = lappend(options, elem); \ } -#define DEFELEM_ADD_INT(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *)makeInteger(value), -1); \ - options = lappend(options, elem); \ +#define DEFELEM_ADD_INT(options, key, value) \ + { \ + DefElem *elem = makeDefElem(key, (Node *) makeInteger(value), -1); \ + options = lappend(options, elem); \ } -static AlterOwnerStmt *RecreateAlterDatabaseOwnerStmt(Oid databaseOid); +static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); -static List *CreateDDLTaskList(char *command, List *workerNodeList, - bool outsideTransaction); +static List * CreateDDLTaskList(char *command, List *workerNodeList, + bool outsideTransaction); PG_FUNCTION_INFO_V1(citus_internal_database_command); static Oid get_database_owner(Oid db_oid); -List *PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); /* controlled via GUC */ bool EnableCreateDatabasePropagation = true; @@ -79,13 +79,14 @@ AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); Assert(stmt->objectType == OBJECT_DATABASE); - Oid databaseOid = get_database_oid(strVal((String *)stmt->object), missing_ok); + Oid databaseOid = get_database_oid(strVal((String *) stmt->object), missing_ok); ObjectAddress *address = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*address, DatabaseRelationId, databaseOid); return list_make1(address); } + /* * DatabaseOwnerDDLCommands returns a list of sql statements to idempotently apply a * change of the database owner on the workers so that the database is owned by the same @@ -94,10 +95,11 @@ AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess) List * DatabaseOwnerDDLCommands(const ObjectAddress *address) { - Node *stmt = (Node *)RecreateAlterDatabaseOwnerStmt(address->objectId); + Node *stmt = (Node *) RecreateAlterDatabaseOwnerStmt(address->objectId); return list_make1(DeparseTreeNode(stmt)); } + /* * RecreateAlterDatabaseOwnerStmt creates an AlterOwnerStmt that represents the operation * of changing the owner of the database to its current owner. @@ -108,7 +110,7 @@ RecreateAlterDatabaseOwnerStmt(Oid databaseOid) AlterOwnerStmt *stmt = makeNode(AlterOwnerStmt); stmt->objectType = OBJECT_DATABASE; - stmt->object = (Node *)makeString(get_database_name(databaseOid)); + stmt->object = (Node *) makeString(get_database_name(databaseOid)); Oid ownerOid = get_database_owner(databaseOid); stmt->newowner = makeNode(RoleSpec); @@ -118,6 +120,7 @@ RecreateAlterDatabaseOwnerStmt(Oid databaseOid) return stmt; } + /* * get_database_owner returns the Oid of the role owning the database */ @@ -131,13 +134,14 @@ get_database_owner(Oid db_oid) errmsg("database with OID %u does not exist", db_oid))); } - Oid dba = ((Form_pg_database)GETSTRUCT(tuple))->datdba; + Oid dba = ((Form_pg_database) GETSTRUCT(tuple))->datdba; ReleaseSysCache(tuple); return dba; } + /* * PreprocessGrantOnDatabaseStmt is executed before the statement is applied to the local * postgres instance. @@ -166,15 +170,16 @@ PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *)stmt); + char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *)sql, + (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + /* * PreprocessAlterDatabaseStmt is executed before the statement is applied to the local * postgres instance. @@ -195,15 +200,16 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *)stmt); + char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *)sql, + (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + #if PG_VERSION_NUM >= PG_VERSION_15 /* @@ -226,15 +232,16 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, EnsureCoordinator(); - char *sql = DeparseTreeNode((Node *)stmt); + char *sql = DeparseTreeNode((Node *) stmt); List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *)sql, + (void *) sql, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + #endif /* @@ -320,7 +327,8 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) * * We do not do this right now because of the AssignDatabaseToShard at the end. */ - List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); + List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, + RowShareLock); if (list_length(workerNodes) > 0) { char *createDatabaseCommand = DeparseTreeNode(node); @@ -345,21 +353,22 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) } /* synchronize pg_dist_object records */ - ObjectAddress dbAddress = {0}; + ObjectAddress dbAddress = { 0 }; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); MarkObjectDistributed(&dbAddress); - return NIL; } + /* * citus_internal_database_command is an internal UDF to * create/drop a database in an idempotent maner without * transaction block restrictions. */ -Datum citus_internal_database_command(PG_FUNCTION_ARGS) +Datum +citus_internal_database_command(PG_FUNCTION_ARGS) { int saveNestLevel = NewGUCNestLevel(); text *commandText = PG_GETARG_TEXT_P(0); @@ -367,7 +376,7 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) Node *parseTree = ParseTreeNode(command); ereport(NOTICE, (errmsg("test internal pre"), - errhint("test pre hint"))); + errhint("test pre hint"))); set_config_option("citus.enable_ddl_propagation", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, @@ -386,7 +395,7 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) if (!OidIsValid(databaseOid)) { - createdb(NULL, (CreatedbStmt *)parseTree); + createdb(NULL, (CreatedbStmt *) parseTree); } else { @@ -407,7 +416,7 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) else { /* remove database from pg_dist_object */ - ObjectAddress dbAddress = {0}; + ObjectAddress dbAddress = { 0 }; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); if (IsObjectDistributed(&dbAddress)) @@ -415,10 +424,10 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) UnmarkObjectDistributed(&dbAddress); } - // /* remove database from database shards */ - // DeleteDatabaseShardByDatabaseIdLocally(databaseOid); + /* / * remove database from database shards * / */ + /* DeleteDatabaseShardByDatabaseIdLocally(databaseOid); */ - DropDatabase(NULL, (DropdbStmt *)parseTree); + DropDatabase(NULL, (DropdbStmt *) parseTree); } } else @@ -431,6 +440,7 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -440,7 +450,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } - DropdbStmt *stmt = (DropdbStmt *)node; + DropdbStmt *stmt = (DropdbStmt *) node; char *databaseName = stmt->dbname; bool missingOk = true; Oid databaseOid = get_database_oid(databaseName, missingOk); @@ -450,14 +460,15 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } - ObjectAddress dbAddress = {0}; + ObjectAddress dbAddress = { 0 }; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); if (!IsObjectDistributed(&dbAddress)) { return NIL; } - List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, RowShareLock); + List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, + RowShareLock); if (list_length(workerNodes) == 0) { return NIL; diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 7465a58cd04..ef7d486b5c0 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -1355,6 +1355,7 @@ GetDistributeObjectOps(Node *node) { return &Database_Alter; } + case T_CreatedbStmt: { return &Database_Create; diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index a818a1ad790..dd729cad0ac 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -694,7 +694,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } /* inform the user about potential caveats */ - if (IsA(parsetree, CreatedbStmt) &&!EnableCreateDatabasePropagation) + if (IsA(parsetree, CreatedbStmt) && !EnableCreateDatabasePropagation) { if (EnableUnsupportedFeatureMessages) { diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 864a9cdf7b6..75ab1d489d6 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -34,7 +34,7 @@ char * DeparseAlterDatabaseOwnerStmt(Node *node) { AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node); - StringInfoData str = {0}; + StringInfoData str = { 0 }; initStringInfo(&str); Assert(stmt->objectType == OBJECT_DATABASE); @@ -44,6 +44,7 @@ DeparseAlterDatabaseOwnerStmt(Node *node) return str.data; } + static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { @@ -51,17 +52,18 @@ AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) appendStringInfo(buf, "ALTER DATABASE %s OWNER TO %s;", - quote_identifier(strVal((String *)stmt->object)), + quote_identifier(strVal((String *) stmt->object)), RoleSpecString(stmt->newowner, true)); } + static void AppendGrantDatabases(StringInfo buf, GrantStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, " ON DATABASE "); - foreach (cell, stmt->objects) + foreach(cell, stmt->objects) { char *database = strVal(lfirst(cell)); appendStringInfoString(buf, quote_identifier(database)); @@ -72,6 +74,7 @@ AppendGrantDatabases(StringInfo buf, GrantStmt *stmt) } } + static void AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt) { @@ -84,12 +87,14 @@ AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt) AppendGrantSharedSuffix(buf, stmt); } + static void AppendDefElemConnLimit(StringInfo buf, DefElem *def) { - appendStringInfo(buf, " CONNECTION LIMIT %ld", (long int)defGetNumeric(def)); + appendStringInfo(buf, " CONNECTION LIMIT %ld", (long int) defGetNumeric(def)); } + static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) { @@ -99,7 +104,7 @@ AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) { ListCell *cell = NULL; appendStringInfo(buf, "WITH "); - foreach (cell, stmt->options) + foreach(cell, stmt->options) { DefElem *def = castNode(DefElem, lfirst(cell)); if (strcmp(def->defname, "is_template") == 0) @@ -128,13 +133,14 @@ AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt) appendStringInfo(buf, ";"); } + char * DeparseGrantOnDatabaseStmt(Node *node) { GrantStmt *stmt = castNode(GrantStmt, node); Assert(stmt->objtype == OBJECT_DATABASE); - StringInfoData str = {0}; + StringInfoData str = { 0 }; initStringInfo(&str); AppendGrantOnDatabaseStmt(&str, stmt); @@ -142,12 +148,13 @@ DeparseGrantOnDatabaseStmt(Node *node) return str.data; } + char * DeparseAlterDatabaseStmt(Node *node) { AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node); - StringInfoData str = {0}; + StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterDatabaseStmt(&str, stmt); @@ -155,11 +162,12 @@ DeparseAlterDatabaseStmt(Node *node) return str.data; } + #if PG_VERSION_NUM >= PG_VERSION_15 char * DeparseAlterDatabaseRefreshCollStmt(Node *node) { - AlterDatabaseRefreshCollStmt *stmt = (AlterDatabaseRefreshCollStmt *)node; + AlterDatabaseRefreshCollStmt *stmt = (AlterDatabaseRefreshCollStmt *) node; StringInfoData str; initStringInfo(&str); @@ -171,6 +179,7 @@ DeparseAlterDatabaseRefreshCollStmt(Node *node) return str.data; } + #endif static void @@ -183,12 +192,13 @@ AppendAlterDatabaseSetStmt(StringInfo buf, AlterDatabaseSetStmt *stmt) AppendVariableSet(buf, varSetStmt); } + char * DeparseAlterDatabaseSetStmt(Node *node) { AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); - StringInfoData str = {0}; + StringInfoData str = { 0 }; initStringInfo(&str); AppendAlterDatabaseSetStmt(&str, stmt); @@ -196,6 +206,7 @@ DeparseAlterDatabaseSetStmt(Node *node) return str.data; } + static void AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -325,11 +336,12 @@ AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) } } + char * DeparseCreateDatabaseStmt(Node *node) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); - StringInfoData str = {0}; + StringInfoData str = { 0 }; initStringInfo(&str); AppendCreatedbStmt(&str, stmt); @@ -337,6 +349,7 @@ DeparseCreateDatabaseStmt(Node *node) return str.data; } + static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { @@ -361,6 +374,7 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) } } + char * DeparseDropDatabaseStmt(Node *node) { @@ -372,6 +386,3 @@ DeparseDropDatabaseStmt(Node *node) return str.data; } - - - diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 862eb0bfc8e..55ec63a6abb 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -2769,8 +2769,6 @@ DistRebalanceStrategyRelationId(void) } - - /* return oid of pg_dist_object relation */ Oid DistObjectRelationId(void) @@ -5403,8 +5401,6 @@ DeformedDistShardTupleToShardInterval(Datum *datumArray, bool *isNullArray, } - - /* * CachedRelationLookup performs a cached lookup for the relation * relationName, with the result cached in *cachedOid. diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 6b764dfbb38..36e41237896 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -42,11 +42,11 @@ extern bool EnforceLocalObjectRestrictions; extern void SwitchToSequentialAndLocalExecutionIfRelationNameTooLong(Oid relationId, char * - finalRelationName); + finalRelationName); extern void SwitchToSequentialAndLocalExecutionIfPartitionNameTooLong(Oid - parentRelationId, + parentRelationId, Oid - partitionRelationId); + partitionRelationId); /* DistOpsOperationType to be used in DistributeObjectOps */ typedef enum DistOpsOperationType @@ -99,7 +99,7 @@ typedef struct DistributeObjectOps #define CITUS_TRUNCATE_TRIGGER_NAME "citus_truncate_trigger" -const DistributeObjectOps *GetDistributeObjectOps(Node *node); +const DistributeObjectOps * GetDistributeObjectOps(Node *node); /* * Flags that can be passed to GetForeignKeyOids to indicate @@ -172,21 +172,23 @@ extern const char *TenantOperationNames[TOTAL_TENANT_OPERATION]; extern void SaveBeginCommandProperties(TransactionStmt *transactionStmt); /* cluster.c - forward declarations */ -extern List *PreprocessClusterStmt(Node *node, const char *clusterCommand, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessClusterStmt(Node *node, const char *clusterCommand, + ProcessUtilityContext processUtilityContext); /* common.c - forward declarations*/ -extern List *PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, - const char *queryString); -extern List *PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, +extern List * PostprocessCreateDistributedObjectFromCatalogStmt(Node *stmt, + const char *queryString); +extern List * PreprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString); +extern List * PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, ProcessUtilityContext - processUtilityContext); -extern List *PostprocessAlterDistributedObjectStmt(Node *stmt, const char *queryString); -extern List *PreprocessDropDistributedObjectStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); + processUtilityContext); +extern List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); @@ -195,78 +197,87 @@ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); extern bool CallDistributedProcedureRemotely(CallStmt *callStmt, DestReceiver *dest); /* collation.c - forward declarations */ -extern char *CreateCollationDDL(Oid collationId); -extern List *CreateCollationDDLsIdempotent(Oid collationId); -extern List *AlterCollationOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *RenameCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *AlterCollationSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern char *GenerateBackupNameForCollationCollision(const ObjectAddress *address); -extern List *DefineCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern char * CreateCollationDDL(Oid collationId); +extern List * CreateCollationDDLsIdempotent(Oid collationId); +extern List * AlterCollationOwnerObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * RenameCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * AlterCollationSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern char * GenerateBackupNameForCollationCollision(const ObjectAddress *address); +extern List * DefineCollationStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); /* database.c - forward declarations */ -extern List *AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *DatabaseOwnerDDLCommands(const ObjectAddress *address); +extern List * AlterDatabaseOwnerObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * DatabaseOwnerDDLCommands(const ObjectAddress *address); -extern List *PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); -extern List *PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); -extern List *PostprocessCreateDatabaseStmt(Node *node, const char *queryString); -extern List *PreprocessDropDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); +extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); - /* domain.c - forward declarations */ - extern List *CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *DomainRenameConstraintStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern CreateDomainStmt *RecreateDomainStmt(Oid domainOid); +/* domain.c - forward declarations */ +extern List * CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterDomainStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * DomainRenameConstraintStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterDomainOwnerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern CreateDomainStmt * RecreateDomainStmt(Oid domainOid); extern Oid get_constraint_typid(Oid conoid); /* extension.c - forward declarations */ extern bool IsDropCitusExtensionStmt(Node *parsetree); -extern List *GetDependentFDWsToExtension(Oid extensionId); +extern List * GetDependentFDWsToExtension(Oid extensionId); extern bool IsCreateAlterExtensionUpdateCitusStmt(Node *parsetree); extern void PreprocessCreateExtensionStmtForCitusColumnar(Node *parsetree); extern void PreprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern void PostprocessAlterExtensionCitusStmtForCitusColumnar(Node *parsetree); extern bool ShouldMarkRelationDistributed(Oid relationId); extern void ErrorIfUnstableCreateOrAlterExtensionStmt(Node *parsetree); -extern List *PostprocessCreateExtensionStmt(Node *stmt, const char *queryString); -extern List *PreprocessDropExtensionStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterExtensionSchemaStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *PostprocessAlterExtensionSchemaStmt(Node *stmt, - const char *queryString); -extern List *PreprocessAlterExtensionUpdateStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List * PostprocessCreateExtensionStmt(Node *stmt, const char *queryString); +extern List * PreprocessDropExtensionStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterExtensionSchemaStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PostprocessAlterExtensionSchemaStmt(Node *stmt, + const char *queryString); +extern List * PreprocessAlterExtensionUpdateStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); extern void PostprocessAlterExtensionCitusUpdateStmt(Node *node); -extern List *PreprocessAlterExtensionContentsStmt(Node *node, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *CreateExtensionDDLCommand(const ObjectAddress *extensionAddress); -extern List *AlterExtensionSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *AlterExtensionUpdateStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); +extern List * PreprocessAlterExtensionContentsStmt(Node *node, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * CreateExtensionDDLCommand(const ObjectAddress *extensionAddress); +extern List * AlterExtensionSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * AlterExtensionUpdateStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); extern void CreateExtensionWithVersion(char *extname, char *extVersion); extern void AlterExtensionUpdateStmt(char *extname, char *extVersion); extern int GetExtensionVersionNumber(char *extVersion); @@ -285,21 +296,21 @@ extern void ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(Oid localTableId); extern bool ColumnReferencedByAnyForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKey(char *columnName, Oid relationId); extern bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid - relationId); -extern List *GetReferencingForeignConstaintCommands(Oid relationOid); -extern List *GetForeignConstraintToReferenceTablesCommands(Oid relationId); -extern List *GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId); -extern List *GetForeignConstraintToDistributedTablesCommands(Oid relationId); -extern List *GetForeignConstraintFromDistributedTablesCommands(Oid relationId); -extern List *GetForeignConstraintCommandsInternal(Oid relationId, int flags); + relationId); +extern List * GetReferencingForeignConstaintCommands(Oid relationOid); +extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId); +extern List * GetForeignConstraintFromOtherReferenceTablesCommands(Oid relationId); +extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId); +extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId); +extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags); extern Oid DropFKeysAndUndistributeTable(Oid relationId); extern void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag); -extern List *GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, - int tableTypeFlag); +extern List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, + int tableTypeFlag); extern bool AnyForeignKeyDependsOnIndex(Oid indexId); extern bool HasForeignKeyWithLocalTable(Oid relationId); extern bool HasForeignKeyToReferenceTable(Oid relationOid); -extern List *GetForeignKeysFromLocalTables(Oid relationId); +extern List * GetForeignKeysFromLocalTables(Oid relationId); extern bool TableReferenced(Oid relationOid); extern bool TableReferencing(Oid relationOid); extern bool ConstraintIsAUniquenessConstraint(char *inputConstaintName, Oid relationId); @@ -308,269 +319,289 @@ extern bool ConstraintWithNameIsOfType(char *inputConstaintName, Oid relationId, char targetConstraintType); extern bool ConstraintWithIdIsOfType(Oid constraintId, char targetConstraintType); extern bool TableHasExternalForeignKeys(Oid relationId); -extern List *GetForeignKeyOids(Oid relationId, int flags); +extern List * GetForeignKeyOids(Oid relationId, int flags); extern Oid GetReferencedTableId(Oid foreignKeyId); extern Oid GetReferencingTableId(Oid foreignKeyId); extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId); /* foreign_data_wrapper.c - forward declarations */ -extern List *PreprocessGrantOnFDWStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern Acl *GetPrivilegesForFDW(Oid FDWOid); +extern List * PreprocessGrantOnFDWStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern Acl * GetPrivilegesForFDW(Oid FDWOid); /* foreign_server.c - forward declarations */ -extern List *PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *GetForeignServerCreateDDLCommand(Oid serverId); +extern List * PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * CreateForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * GetForeignServerCreateDDLCommand(Oid serverId); /* foreign_table.c - forward declarations */ -extern List *PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List * PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); /* function.c - forward declarations */ -extern List *PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, +extern List * PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessCreateFunctionStmt(Node *stmt, + const char *queryString); +extern List * CreateFunctionStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * DefineAggregateStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * PreprocessAlterFunctionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PostprocessCreateFunctionStmt(Node *stmt, - const char *queryString); -extern List *CreateFunctionStmtObjectAddress(Node *stmt, +extern List * AlterFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *DefineAggregateStmtObjectAddress(Node *stmt, +extern List * RenameFunctionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *PreprocessAlterFunctionStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *AlterFunctionStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *RenameFunctionStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *AlterFunctionOwnerObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *AlterFunctionSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *PreprocessAlterFunctionDependsStmt(Node *stmt, - const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *AlterFunctionDependsStmtObjectAddress(Node *stmt, +extern List * AlterFunctionOwnerObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * AlterFunctionSchemaStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); +extern List * PreprocessAlterFunctionDependsStmt(Node *stmt, + const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * AlterFunctionDependsStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); /* grant.c - forward declarations */ -extern List *PreprocessGrantStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessGrantStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void deparsePrivileges(StringInfo privsString, GrantStmt *grantStmt); extern void deparseGrantees(StringInfo granteesString, GrantStmt *grantStmt); /* index.c - forward declarations */ extern bool IsIndexRenameStmt(RenameStmt *renameStmt); -extern List *PreprocessIndexStmt(Node *createIndexStatement, - const char *createIndexCommand, - ProcessUtilityContext processUtilityContext); -extern char *ChooseIndexName(const char *tabname, Oid namespaceId, - List *colnames, List *exclusionOpNames, - bool primary, bool isconstraint); -extern char *ChooseIndexNameAddition(List *colnames); -extern List *ChooseIndexColumnNames(List *indexElems); +extern List * PreprocessIndexStmt(Node *createIndexStatement, + const char *createIndexCommand, + ProcessUtilityContext processUtilityContext); +extern char * ChooseIndexName(const char *tabname, Oid namespaceId, + List *colnames, List *exclusionOpNames, + bool primary, bool isconstraint); +extern char * ChooseIndexNameAddition(List *colnames); +extern List * ChooseIndexColumnNames(List *indexElems); extern LOCKMODE GetCreateIndexRelationLockMode(IndexStmt *createIndexStatement); -extern List *PreprocessReindexStmt(Node *ReindexStatement, - const char *ReindexCommand, - ProcessUtilityContext processUtilityContext); -extern List *ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *PreprocessDropIndexStmt(Node *dropIndexStatement, - const char *dropIndexCommand, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessIndexStmt(Node *node, - const char *queryString); +extern List * PreprocessReindexStmt(Node *ReindexStatement, + const char *ReindexCommand, + ProcessUtilityContext processUtilityContext); +extern List * ReindexStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List * PreprocessDropIndexStmt(Node *dropIndexStatement, + const char *dropIndexCommand, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessIndexStmt(Node *node, + const char *queryString); extern void ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement); extern void MarkIndexValid(IndexStmt *indexStmt); -extern List *ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcessor, int flags); +extern List * ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor + pgIndexProcessor, int flags); extern bool IsReindexWithParam_compat(ReindexStmt *stmt, char *paramName); /* objectaddress.c - forward declarations */ -extern List *CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List * CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); /* owned.c - forward declarations */ -extern List *PreprocessDropOwnedStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessDropOwnedStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); /* policy.c - forward declarations */ -extern List *CreatePolicyCommands(Oid relationId); +extern List * CreatePolicyCommands(Oid relationId); extern void ErrorIfUnsupportedPolicy(Relation relation); extern void ErrorIfUnsupportedPolicyExpr(Node *expr); -extern List *PostprocessCreatePolicyStmt(Node *node, const char *queryString); -extern List *PreprocessAlterPolicyStmt(Node *node, const char *queryString, +extern List * PostprocessCreatePolicyStmt(Node *node, const char *queryString); +extern List * PreprocessAlterPolicyStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessDropPolicyStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PreprocessDropPolicyStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); extern bool IsPolicyRenameStmt(RenameStmt *stmt); extern void CreatePolicyEventExtendNames(CreatePolicyStmt *stmt, const char *schemaName, uint64 shardId); extern void AlterPolicyEventExtendNames(AlterPolicyStmt *stmt, const char *schemaName, uint64 shardId); -extern void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 shardId); -extern void DropPolicyEventExtendNames(DropStmt *stmt, const char *schemaName, uint64 shardId); +extern void RenamePolicyEventExtendNames(RenameStmt *stmt, const char *schemaName, uint64 + shardId); +extern void DropPolicyEventExtendNames(DropStmt *stmt, const char *schemaName, uint64 + shardId); extern void AddRangeTableEntryToQueryCompat(ParseState *parseState, Relation relation); /* publication.c - forward declarations */ -extern List *PostProcessCreatePublicationStmt(Node *node, const char *queryString); -extern List *CreatePublicationDDLCommandsIdempotent(const ObjectAddress *address); -extern char *CreatePublicationDDLCommand(Oid publicationId); -extern List *PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityCtx); -extern List *GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd); -extern char *GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, - bool isAdd); -extern List *AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); -extern List *AlterPublicationStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); -extern List *CreatePublicationStmtObjectAddress(Node *node, bool missingOk, - bool isPostProcess); -extern List *RenamePublicationStmtObjectAddress(Node *node, bool missingOk, +extern List * PostProcessCreatePublicationStmt(Node *node, const char *queryString); +extern List * CreatePublicationDDLCommandsIdempotent(const ObjectAddress *address); +extern char * CreatePublicationDDLCommand(Oid publicationId); +extern List * PreprocessAlterPublicationStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityCtx); +extern List * GetAlterPublicationDDLCommandsForTable(Oid relationId, bool isAdd); +extern char * GetAlterPublicationTableDDLCommand(Oid publicationId, Oid relationId, + bool isAdd); +extern List * AlterPublicationOwnerStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); +extern List * AlterPublicationStmtObjectAddress(Node *node, bool missingOk, bool isPostProcess); +extern List * CreatePublicationStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); +extern List * RenamePublicationStmtObjectAddress(Node *node, bool missingOk, + bool isPostProcess); /* rename.c - forward declarations*/ -extern List *PreprocessRenameStmt(Node *renameStmt, const char *renameCommand, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessRenameStmt(Node *renameStmt, const char *renameCommand, + ProcessUtilityContext processUtilityContext); extern void ErrorIfUnsupportedRenameStmt(RenameStmt *renameStmt); -extern List *PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); /* role.c - forward declarations*/ -extern List *PostprocessAlterRoleStmt(Node *stmt, const char *queryString); -extern List *PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PostprocessAlterRoleStmt(Node *stmt, const char *queryString); +extern List * PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); extern List * PreprocessAlterRoleRenameStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *GenerateAlterRoleSetCommandForRole(Oid roleid); -extern List *AlterRoleStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterRoleSetStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *PreprocessCreateRoleStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessDropRoleStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessGrantRoleStmt(Node *stmt, const char *queryString, +extern List * GenerateAlterRoleSetCommandForRole(Oid roleid); +extern List * AlterRoleStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List * AlterRoleSetStmtObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List * PreprocessCreateRoleStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessDropRoleStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PostprocessGrantRoleStmt(Node *stmt, const char *queryString); -extern List *GenerateCreateOrAlterRoleCommand(Oid roleOid); -extern List *CreateRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List * PreprocessGrantRoleStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessGrantRoleStmt(Node *stmt, const char *queryString); +extern List * GenerateCreateOrAlterRoleCommand(Oid roleOid); +extern List * CreateRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); extern List * RenameRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); extern void UnmarkRolesDistributed(List *roles); -extern List *FilterDistributedRoles(List *roles); +extern List * FilterDistributedRoles(List *roles); /* schema.c - forward declarations */ -extern List *PostprocessCreateSchemaStmt(Node *node, const char *queryString); -extern List *PreprocessDropSchemaStmt(Node *dropSchemaStatement, - const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterObjectSchemaStmt(Node *alterObjectSchemaStmt, - const char *alterObjectSchemaCommand); -extern List *PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, - bool isPostprocess); -extern List *AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * PostprocessCreateSchemaStmt(Node *node, const char *queryString); +extern List * PreprocessDropSchemaStmt(Node *dropSchemaStatement, + const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterObjectSchemaStmt(Node *alterObjectSchemaStmt, + const char *alterObjectSchemaCommand); +extern List * PreprocessGrantOnSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * CreateSchemaStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterSchemaOwnerStmtObjectAddress(Node *node, bool missing_ok, + bool isPostprocess); +extern List * AlterSchemaRenameStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); /* sequence.c - forward declarations */ -extern List *PreprocessAlterSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString); -extern List *PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); +extern List * PreprocessAlterSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterSequenceSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PostprocessAlterSequenceSchemaStmt(Node *node, const char *queryString); +extern List * PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString); #if (PG_VERSION_NUM >= PG_VERSION_15) -extern List *PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterSequencePersistenceStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PreprocessSequenceAlterTableStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); #endif -extern List *PreprocessDropSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *PreprocessRenameSequenceStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, +extern List * PreprocessDropSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * SequenceDropStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * PreprocessRenameSequenceStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PostprocessGrantOnSequenceStmt(Node *node, const char *queryString); -extern List *AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * PreprocessGrantOnSequenceStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessGrantOnSequenceStmt(Node *node, const char *queryString); +extern List * AlterSequenceStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterSequenceSchemaStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); #if (PG_VERSION_NUM >= PG_VERSION_15) -extern List *AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * AlterSequencePersistenceStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); #endif -extern List *RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * RenameSequenceStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); extern void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt); extern void ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt); -extern char *GenerateBackupNameForSequenceCollision(const ObjectAddress *address); +extern char * GenerateBackupNameForSequenceCollision(const ObjectAddress *address); extern void RenameExistingSequenceWithDifferentTypeIfExists(RangeVar *sequence, Oid desiredSeqTypeId); /* statistics.c - forward declarations */ -extern List *PreprocessCreateStatisticsStmt(Node *node, const char *queryString, +extern List * PreprocessCreateStatisticsStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessCreateStatisticsStmt(Node *node, const char *queryString); +extern List * CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool + isPostprocess); +extern List * PreprocessDropStatisticsStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * DropStatisticsObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern List * PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString); +extern List * AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool + isPostprocess); +extern List * PreprocessAlterStatisticsStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PostprocessCreateStatisticsStmt(Node *node, const char *queryString); -extern List *CreateStatisticsStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); -extern List *PreprocessDropStatisticsStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *DropStatisticsObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *PreprocessAlterStatisticsRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *PreprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString, +extern List * PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext - processUtilityContext); -extern List *PostprocessAlterStatisticsSchemaStmt(Node *node, const char *queryString); -extern List *AlterStatisticsSchemaStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); -extern List *PreprocessAlterStatisticsStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern List *PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString); -extern List *GetExplicitStatisticsCommandList(Oid relationId); -extern List *GetExplicitStatisticsSchemaIdList(Oid relationId); -extern List *GetAlterIndexStatisticsCommands(Oid indexOid); + processUtilityContext); +extern List * PostprocessAlterStatisticsOwnerStmt(Node *node, const char *queryString); +extern List * GetExplicitStatisticsCommandList(Oid relationId); +extern List * GetExplicitStatisticsSchemaIdList(Oid relationId); +extern List * GetAlterIndexStatisticsCommands(Oid indexOid); /* subscription.c - forward declarations */ -extern Node *ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); +extern Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); /* table.c - forward declarations */ -extern List *PreprocessDropTableStmt(Node *stmt, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessDropTableStmt(Node *stmt, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString); extern bool ShouldEnableLocalReferenceForeignKeys(void); -extern List *PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, - const char *queryString); -extern List *PostprocessAlterTableSchemaStmt(Node *node, const char *queryString); +extern List * PreprocessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement, + const char *queryString); +extern List * PostprocessAlterTableSchemaStmt(Node *node, const char *queryString); extern void PrepareAlterTableStmtForConstraint(AlterTableStmt *alterTableStatement, Oid relationId, Constraint *constraint); -extern List *PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, - ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, +extern List * PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PreprocessAlterTableSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt, bool processLocalRelation); extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt); @@ -582,144 +613,160 @@ extern void ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *c extern void ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod, char referencingReplicationModel, Var *distributionColumn, uint32 colocationId); -extern List *InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, - const char *commandString); -extern List *AlterTableSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *MakeNameListFromRangeVar(const RangeVar *rel); +extern List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId, + const char *commandString); +extern List * AlterTableSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * MakeNameListFromRangeVar(const RangeVar *rel); extern Oid GetSequenceOid(Oid relationId, AttrNumber attnum); extern bool ConstrTypeUsesIndex(ConstrType constrType); extern bool ConstrTypeCitusCanDefaultName(ConstrType constrType); -extern char *GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, - char *colname, bool missingTableOk); +extern char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId, + char *colname, bool missingTableOk); extern void ErrorIfTableHasIdentityColumn(Oid relationId); extern void ConvertNewTableIfNecessary(Node *createStmt); extern void ConvertToTenantTableIfNecessary(AlterObjectSchemaStmt *alterObjectSchemaStmt); /* text_search.c - forward declarations */ -extern List *GetCreateTextSearchConfigStatements(const ObjectAddress *address); -extern List *GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); -extern List *CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address); -extern List *CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address); -extern List *CreateTextSearchConfigurationObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *CreateTextSearchDictObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *RenameTextSearchConfigurationStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *RenameTextSearchDictionaryStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchConfigurationStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchDictionaryStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *TextSearchConfigurationCommentObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *TextSearchDictCommentObjectAddress(Node *node, +extern List * GetCreateTextSearchConfigStatements(const ObjectAddress *address); +extern List * GetCreateTextSearchDictionaryStatements(const ObjectAddress *address); +extern List * CreateTextSearchConfigDDLCommandsIdempotent(const ObjectAddress *address); +extern List * CreateTextSearchDictDDLCommandsIdempotent(const ObjectAddress *address); +extern List * CreateTextSearchConfigurationObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * CreateTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchConfigurationOwnerObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern List *AlterTextSearchDictOwnerObjectAddress(Node *node, - bool missing_ok, bool isPostprocess); -extern char *GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address); -extern char *GenerateBackupNameForTextSearchDict(const ObjectAddress *address); -extern List *get_ts_config_namelist(Oid tsconfigOid); +extern List * RenameTextSearchConfigurationStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * RenameTextSearchDictionaryStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterTextSearchConfigurationStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterTextSearchDictionaryStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterTextSearchConfigurationSchemaStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterTextSearchDictionarySchemaStmtObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * TextSearchConfigurationCommentObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * TextSearchDictCommentObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern List * AlterTextSearchConfigurationOwnerObjectAddress(Node *node, + bool missing_ok, bool + isPostprocess); +extern List * AlterTextSearchDictOwnerObjectAddress(Node *node, + bool missing_ok, bool isPostprocess); +extern char * GenerateBackupNameForTextSearchConfiguration(const ObjectAddress *address); +extern char * GenerateBackupNameForTextSearchDict(const ObjectAddress *address); +extern List * get_ts_config_namelist(Oid tsconfigOid); /* truncate.c - forward declarations */ extern void PreprocessTruncateStatement(TruncateStmt *truncateStatement); /* type.c - forward declarations */ -extern List *PreprocessRenameTypeAttributeStmt(Node *stmt, const char *queryString, - ProcessUtilityContext - processUtilityContext); -extern Node *CreateTypeStmtByObjectAddress(const ObjectAddress *address); -extern List *CompositeTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *CreateEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *AlterTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *AlterEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *RenameTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *AlterTypeSchemaStmtObjectAddress(Node *stmt, - bool missing_ok, bool isPostprocess); -extern List *RenameTypeAttributeStmtObjectAddress(Node *stmt, - bool missing_ok); -extern List *AlterTypeOwnerObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); -extern List *CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress); -extern char *GenerateBackupNameForTypeCollision(const ObjectAddress *address); +extern List * PreprocessRenameTypeAttributeStmt(Node *stmt, const char *queryString, + ProcessUtilityContext + processUtilityContext); +extern Node * CreateTypeStmtByObjectAddress(const ObjectAddress *address); +extern List * CompositeTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * CreateEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * AlterTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List * AlterEnumStmtObjectAddress(Node *stmt, bool missing_ok, bool isPostprocess); +extern List * RenameTypeStmtObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * AlterTypeSchemaStmtObjectAddress(Node *stmt, + bool missing_ok, bool isPostprocess); +extern List * RenameTypeAttributeStmtObjectAddress(Node *stmt, + bool missing_ok); +extern List * AlterTypeOwnerObjectAddress(Node *stmt, bool missing_ok, bool + isPostprocess); +extern List * CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress); +extern char * GenerateBackupNameForTypeCollision(const ObjectAddress *address); /* function.c - forward declarations */ -extern List *CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress); -extern char *GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); -extern char *GenerateBackupNameForProcCollision(const ObjectAddress *address); -extern ObjectWithArgs *ObjectWithArgsFromOid(Oid funcOid); +extern List * CreateFunctionDDLCommandsIdempotent(const ObjectAddress *functionAddress); +extern char * GetFunctionDDLCommand(const RegProcedure funcOid, bool useCreateOrReplace); +extern char * GenerateBackupNameForProcCollision(const ObjectAddress *address); +extern ObjectWithArgs * ObjectWithArgsFromOid(Oid funcOid); extern void UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, int *distribution_argument_index, int *colocationId, bool *forceDelegation); /* vacuum.c - forward declarations */ -extern List *PostprocessVacuumStmt(Node *node, const char *vacuumCommand); +extern List * PostprocessVacuumStmt(Node *node, const char *vacuumCommand); /* view.c - forward declarations */ -extern List *PreprocessViewStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessViewStmt(Node *node, const char *queryString); -extern List *ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *PreprocessDropViewStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *DropViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern char *CreateViewDDLCommand(Oid viewOid); -extern List *GetViewCreationCommandsOfTable(Oid relationId); -extern char *AlterViewOwnerCommand(Oid viewOid); -extern char *DeparseViewStmt(Node *node); -extern char *DeparseDropViewStmt(Node *node); -extern bool IsViewDistributed(Oid viewOid); -extern List *CreateViewDDLCommandsIdempotent(Oid viewOid); -extern List *PreprocessAlterViewStmt(Node *node, const char *queryString, +extern List * PreprocessViewStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessViewStmt(Node *node, const char *queryString); +extern List * ViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * AlterViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * PreprocessDropViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *PostprocessAlterViewStmt(Node *node, const char *queryString); -extern List *PreprocessRenameViewStmt(Node *node, const char *queryString, +extern List * DropViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern char * CreateViewDDLCommand(Oid viewOid); +extern List * GetViewCreationCommandsOfTable(Oid relationId); +extern char * AlterViewOwnerCommand(Oid viewOid); +extern char * DeparseViewStmt(Node *node); +extern char * DeparseDropViewStmt(Node *node); +extern bool IsViewDistributed(Oid viewOid); +extern List * CreateViewDDLCommandsIdempotent(Oid viewOid); +extern List * PreprocessAlterViewStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List *RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List *PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessAlterViewSchemaStmt(Node *node, const char *queryString); -extern List *AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * PostprocessAlterViewStmt(Node *node, const char *queryString); +extern List * PreprocessRenameViewStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * RenameViewStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * PreprocessAlterViewSchemaStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessAlterViewSchemaStmt(Node *node, const char *queryString); +extern List * AlterViewSchemaStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); extern bool IsViewRenameStmt(RenameStmt *renameStmt); /* trigger.c - forward declarations */ -extern List *GetExplicitTriggerCommandList(Oid relationId); +extern List * GetExplicitTriggerCommandList(Oid relationId); extern HeapTuple GetTriggerTupleById(Oid triggerId, bool missingOk); -extern List *GetExplicitTriggerIdList(Oid relationId); -extern List *PostprocessCreateTriggerStmt(Node *node, const char *queryString); -extern List *CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); +extern List * GetExplicitTriggerIdList(Oid relationId); +extern List * PostprocessCreateTriggerStmt(Node *node, const char *queryString); +extern List * CreateTriggerStmtObjectAddress(Node *node, bool missingOk, bool + isPostprocess); extern void CreateTriggerEventExtendNames(CreateTrigStmt *createTriggerStmt, char *schemaName, uint64 shardId); -extern List *PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); -extern List *PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); +extern List * PreprocessAlterTriggerRenameStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); +extern List * PostprocessAlterTriggerRenameStmt(Node *node, const char *queryString); extern void AlterTriggerRenameEventExtendNames(RenameStmt *renameTriggerStmt, char *schemaName, uint64 shardId); -extern List *PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString); -extern List *PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, - ProcessUtilityContext - processUtilityContext); +extern List * PostprocessAlterTriggerDependsStmt(Node *node, const char *queryString); +extern List * PreprocessAlterTriggerDependsStmt(Node *node, const char *queryString, + ProcessUtilityContext + processUtilityContext); extern void AlterTriggerDependsEventExtendNames( AlterObjectDependsStmt *alterTriggerDependsStmt, char *schemaName, uint64 shardId); extern void ErrorOutForTriggerIfNotSupported(Oid relationId); extern void ErrorIfRelationHasUnsupportedTrigger(Oid relationId); -extern List *PreprocessDropTriggerStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); +extern List * PreprocessDropTriggerStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern void DropTriggerEventExtendNames(DropStmt *dropTriggerStmt, char *schemaName, uint64 shardId); -extern List *CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, - const char *queryString); +extern List * CitusCreateTriggerCommandDDLJob(Oid relationId, char *triggerName, + const char *queryString); extern Oid GetTriggerFunctionId(Oid triggerId); /* cascade_table_operation_for_connected_relations.c */ @@ -746,12 +793,12 @@ typedef enum CascadeOperationType extern void CascadeOperationForFkeyConnectedRelations(Oid relationId, LOCKMODE relLockMode, CascadeOperationType - cascadeOperationType); + cascadeOperationType); extern void CascadeOperationForRelationIdList(List *relationIdList, LOCKMODE lockMode, CascadeOperationType cascadeOperationType); extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList); extern bool RelationIdListHasReferenceTable(List *relationIdList); -extern List *GetFKeyCreationCommandsForRelationIdList(List *relationIdList); +extern List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList); extern void DropRelationForeignKeys(Oid relationId, int flags); extern void SetLocalEnableLocalReferenceForeignKeys(bool state); extern void ExecuteAndLogUtilityCommandListInTableTypeConversionViaSPI( @@ -765,7 +812,7 @@ extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList, extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConverted); extern bool ShouldAddNewTableToMetadata(Oid relationId); -extern List *GetExplicitIndexOidList(Oid relationId); +extern List * GetExplicitIndexOidList(Oid relationId); extern bool ShouldPropagateSetCommand(VariableSetStmt *setStmt); extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *setCommand); @@ -773,7 +820,7 @@ extern void PostprocessVariableSetStmt(VariableSetStmt *setStmt, const char *set extern void CreateCitusLocalTablePartitionOf(CreateStmt *createStatement, Oid relationId, Oid parentRelationId); extern void UpdateAutoConvertedForConnectedRelations(List *relationId, bool - autoConverted); + autoConverted); /* schema_based_sharding.c */ extern bool ShouldUseSchemaBasedSharding(char *schemaName); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index f7ba8c02b4d..6d76b03dd7c 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -231,7 +231,6 @@ extern char * DeparseCreateDatabaseStmt(Node *node); extern char * DeparseDropDatabaseStmt(Node *node); - /* forward declaration for deparse_publication_stmts.c */ extern char * DeparseCreatePublicationStmt(Node *stmt); extern char * DeparseCreatePublicationStmtExtended(Node *node, From 3735bef156c108215e2f4e4431adaf32a4e298c6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 13:40:37 +0300 Subject: [PATCH 009/180] Fixes sql comments --- src/backend/distributed/sql/citus--12.1-1--12.2-1.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql index 94814c48be6..5d869f40e0d 100644 --- a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql @@ -1,8 +1,8 @@ -- citus--12.1-1--12.2-1 -/* - * citus_internal_database_command creates a database according to the given command. - */ +-- +-- citus_internal_database_command creates a database according to the given command. + CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) RETURNS void LANGUAGE C From 5c50d5875756a4b62495c5613ca8e97c5cd5569d Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 15:06:22 +0300 Subject: [PATCH 010/180] Changes EnableCreateDatabasePropagation to false to fix failing tests --- src/backend/distributed/shared_library_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 886517464e5..00cdb0027e2 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1268,7 +1268,7 @@ RegisterCitusConfigVariables(void) "and DROP DATABASE statements to workers"), NULL, &EnableCreateDatabasePropagation, - true, + false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); From 8c98369e215d2166cc86a4902544db7c2e45333d Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 15:33:14 +0300 Subject: [PATCH 011/180] Fixes affected tests --- src/test/regress/expected/multi_extension.out | 3 ++- src/test/regress/expected/upgrade_list_citus_objects.out | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 295b10c76e0..43f9c3b983c 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1422,7 +1422,8 @@ ALTER EXTENSION citus UPDATE TO '12.2-1'; SELECT * FROM multi_extension.print_extension_changes(); previous_object | current_object --------------------------------------------------------------------- -(0 rows) + | function citus_internal_database_command(text) void +(1 row) DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff; -- show running version diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 36bd504e88d..942e0336f61 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -71,6 +71,7 @@ ORDER BY 1; function citus_internal_add_shard_metadata(regclass,bigint,"char",text,text) function citus_internal_add_tenant_schema(oid,integer) function citus_internal_adjust_local_clock_to_remote(cluster_clock) + function citus_internal_database_command(text) function citus_internal_delete_colocation_metadata(integer) function citus_internal_delete_partition_metadata(regclass) function citus_internal_delete_placement_metadata(bigint) @@ -343,5 +344,5 @@ ORDER BY 1; view citus_stat_tenants_local view pg_dist_shard_placement view time_partitions -(333 rows) +(334 rows) From a5e0b9411a9394dd419a00fac72fd613f3776dd9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 21:20:19 +0300 Subject: [PATCH 012/180] Adds tests for create and drop database --- src/backend/distributed/commands/database.c | 5 ++ .../deparser/deparse_database_stmts.c | 11 +++-- .../create_drop_database_propagation.out | 46 +++++++++++++++++++ src/test/regress/multi_1_schedule | 1 + .../sql/create_drop_database_propagation.sql | 45 ++++++++++++++++++ 5 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 src/test/regress/expected/create_drop_database_propagation.out create mode 100644 src/test/regress/sql/create_drop_database_propagation.sql diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index c760aaa705b..c7da5d06889 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -312,6 +312,11 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) { + if (EnableCreateDatabasePropagation) + { + EnsureCoordinator(); + } + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 75ab1d489d6..6ffce3b4832 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -274,6 +274,13 @@ AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) appendStringInfo(buf, " ICU_LOCALE %s", quote_literal_cstr(icuLocale)); } + else if (strcmp(option->defname, "icu_rules") == 0) + { + char *icuLocale = defGetString(option); + + appendStringInfo(buf, " ICU_RULES %s", + quote_literal_cstr(icuLocale)); + } else if (strcmp(option->defname, "locale_provider") == 0) { char *localeProvider = defGetString(option); @@ -323,10 +330,6 @@ AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) quote_literal_cstr(strategy)); } #endif - else if (strcmp(option->defname, "location") == 0) - { - /* deprecated option */ - } else { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out new file mode 100644 index 00000000000..34251f9c4ed --- /dev/null +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -0,0 +1,46 @@ +set citus.enable_create_database_propagation=on; +create user create_drop_db_test_user; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.UTF-8' + LC_COLLATE = 'en_US.UTF-8' + LC_CTYPE = 'en_US.UTF-8' + ICU_LOCALE = 'en_US.UTF-8' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = test_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 17343 | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace +(1 row) + +drop database mydatabase; +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +drop user create_drop_db_test_user; +set citus.enable_create_database_propagation=off; diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index ad70f136e5a..6cd6225ef42 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -52,6 +52,7 @@ test: multi_read_from_secondaries test: grant_on_database_propagation test: alter_database_propagation +test: create_drop_database_propagation test: citus_shards diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql new file mode 100644 index 00000000000..27820b32bbf --- /dev/null +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -0,0 +1,45 @@ +set citus.enable_create_database_propagation=on; +create user create_drop_db_test_user; + + +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.UTF-8' + LC_COLLATE = 'en_US.UTF-8' + LC_CTYPE = 'en_US.UTF-8' + ICU_LOCALE = 'en_US.UTF-8' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = test_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +drop database mydatabase; + +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +drop user create_drop_db_test_user; +set citus.enable_create_database_propagation=off; + + From 25488afccd564a0016eae1fbcdd18201b1f4cb05 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 4 Oct 2023 23:11:48 +0300 Subject: [PATCH 013/180] Changes encoding --- .../create_drop_database_propagation.out | 32 +++++++++---------- .../sql/create_drop_database_propagation.sql | 28 ++++++++-------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 34251f9c4ed..5b835567ea2 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -2,20 +2,20 @@ set citus.enable_create_database_propagation=on; create user create_drop_db_test_user; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.UTF-8' - LC_COLLATE = 'en_US.UTF-8' - LC_CTYPE = 'en_US.UTF-8' - ICU_LOCALE = 'en_US.UTF-8' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = test_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = test_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, @@ -24,9 +24,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17343 | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 17343 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace (1 row) drop database mydatabase; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 27820b32bbf..487f1382644 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -4,20 +4,20 @@ create user create_drop_db_test_user; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.UTF-8' - LC_COLLATE = 'en_US.UTF-8' - LC_CTYPE = 'en_US.UTF-8' - ICU_LOCALE = 'en_US.UTF-8' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = test_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = test_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, From d0bd20c9b1f7d385b58ce2ded6219ab4a4eaefc6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 17:11:07 +0300 Subject: [PATCH 014/180] Fixes tests --- .../create_drop_database_propagation.out | 101 +++++++++-- src/test/regress/expected/pg15.out | 168 +++++++++++++----- src/test/regress/pg_regress_multi.pl | 2 +- .../sql/create_drop_database_propagation.sql | 96 ++++++++-- src/test/regress/sql/pg15.sql | 112 ++++++++++++ 5 files changed, 400 insertions(+), 79 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 5b835567ea2..98155024ce1 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -1,46 +1,111 @@ -set citus.enable_create_database_propagation=on; +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'C' LC_COLLATE = 'C' LC_CTYPE = 'C' - ICU_LOCALE = 'C' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = test_tablespace ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, + IS_TEMPLATE = false; +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 17344 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace +(1 row) + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 17404 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace +(1 row) + +\c - - - :worker_2_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17343 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 17404 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace (1 row) +\c - - - :master_port +set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_2_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) +\c - - - :master_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; +\c - - - :master_port drop user create_drop_db_test_user; -set citus.enable_create_database_propagation=off; diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index fcbb0cd1220..2f005070a9c 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -586,59 +586,19 @@ WHERE f.numeric_column = s.numeric_column; -- test new regex functions -- print order comments that contain the word `fluffily` at least twice SELECT o_comment FROM public.orders WHERE regexp_count(o_comment, 'FluFFily', 1, 'i')>=2 ORDER BY 1; - o_comment ---------------------------------------------------------------------- - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli - ly regular packages are fluffily even ideas. fluffily final - ng instructions integrate fluffily among the fluffily silent accounts. bli - ructions wake fluffily fluffily final gifts! furiou - s boost blithely fluffily idle ideas? fluffily even pin -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the same items using a different regexp function SELECT o_comment FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1; - o_comment ---------------------------------------------------------------------- - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli - ly regular packages are fluffily even ideas. fluffily final - ng instructions integrate fluffily among the fluffily silent accounts. bli - ructions wake fluffily fluffily final gifts! furiou - s boost blithely fluffily idle ideas? fluffily even pin -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the position where we find the second fluffily in the comment SELECT o_comment, regexp_instr(o_comment, 'fluffily.*(fluffily)') FROM public.orders ORDER BY 2 desc LIMIT 5; - o_comment | regexp_instr ---------------------------------------------------------------------- - ng instructions integrate fluffily among the fluffily silent accounts. bli | 27 - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli | 26 - ly regular packages are fluffily even ideas. fluffily final | 25 - s boost blithely fluffily idle ideas? fluffily even pin | 18 - ructions wake fluffily fluffily final gifts! furiou | 15 -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the substrings between two `fluffily` SELECT regexp_substr(o_comment, 'fluffily.*fluffily') FROM public.orders ORDER BY 1 LIMIT 5; - regexp_substr ---------------------------------------------------------------------- - fluffily among the fluffily - fluffily even ideas. fluffily - fluffily fluffily - fluffily fluffily - fluffily idle ideas? fluffily -(5 rows) - +ERROR: relation "public.orders" does not exist -- replace second `fluffily` with `silkily` SELECT regexp_replace(o_comment, 'fluffily', 'silkily', 1, 2) FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1 desc; - regexp_replace ---------------------------------------------------------------------- - s boost blithely fluffily idle ideas? silkily even pin - ructions wake fluffily silkily final gifts! furiou - ng instructions integrate fluffily among the silkily silent accounts. bli - ly regular packages are fluffily even ideas. silkily final - al, bold deposits cajole fluffily silkily final foxes. pending ideas beli -(5 rows) - +ERROR: relation "public.orders" does not exist -- test new COPY features -- COPY TO statements with text format and headers CREATE TABLE copy_test(id int, data int); @@ -1536,3 +1496,121 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; +-- create/drop database for pg > 15 +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 17851 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 18201 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :worker_2_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 18174 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :master_port +set citus.enable_create_database_propagation=on; +drop database mydatabase; +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_2_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; +\c - - - :master_port +drop user create_drop_db_test_user; diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 4cc022198c4..ba2251e37e2 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -630,7 +630,7 @@ sub generate_hba } } -for my $tablespace ("ts0", "ts1", "ts2") +for my $tablespace ("ts0", "ts1", "ts2", "ts3", "ts4", "ts5") { if (-e catfile($TMP_CHECKDIR, $tablespace)) { diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 487f1382644..cfcd21e6708 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -1,45 +1,111 @@ -set citus.enable_create_database_propagation=on; + + +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :master_port create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'C' LC_COLLATE = 'C' LC_CTYPE = 'C' - ICU_LOCALE = 'C' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = test_tablespace ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; + IS_TEMPLATE = false; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname , pd.datdba, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; +\c - - - :worker_1_port + +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; +\c - - - :worker_2_port + +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :master_port +set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname , pd.datdba, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; +\c - - - :worker_1_port + +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :worker_2_port + +SELECT pd.datname , pd.datdba, pd.encoding, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :master_port + +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_1_port + +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_2_port + +drop tablespace create_drop_db_tablespace; + +\c - - - :master_port + drop user create_drop_db_test_user; -set citus.enable_create_database_propagation=off; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index fe60222ddc4..37270771b72 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -976,3 +976,115 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; + + +-- create/drop database for pg > 15 + + +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :worker_2_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :master_port +set citus.enable_create_database_propagation=on; +drop database mydatabase; +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :worker_1_port +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :worker_2_port + +SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; + +\c - - - :master_port +drop user create_drop_db_test_user; From b01c9ffb45d0f48df1f1b82db4c8af94ac173f86 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 17:27:07 +0300 Subject: [PATCH 015/180] Removes unnecessary fileds from selects --- .../create_drop_database_propagation.out | 30 +++++++++---------- src/test/regress/expected/pg15.out | 30 +++++++++---------- .../sql/create_drop_database_propagation.sql | 12 ++++---- src/test/regress/sql/pg15.sql | 12 ++++---- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 98155024ce1..3321dd28996 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -19,7 +19,7 @@ CREATE DATABASE mydatabase TABLESPACE = test_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -27,13 +27,13 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17344 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace (1 row) \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -41,13 +41,13 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17404 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace (1 row) \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -55,15 +55,15 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17404 | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace (1 row) \c - - - :master_port set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -71,12 +71,12 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -84,12 +84,12 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -97,7 +97,7 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 2f005070a9c..c0e510cccd9 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1524,7 +1524,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false OID = 966345; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1532,13 +1532,13 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 17851 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1546,13 +1546,13 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 18201 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1560,15 +1560,15 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 18174 | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1576,12 +1576,12 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1589,12 +1589,12 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1602,7 +1602,7 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | datdba | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index cfcd21e6708..39344c7d8ba 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -27,7 +27,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -38,7 +38,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -48,7 +48,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -61,7 +61,7 @@ WHERE datname = 'mydatabase'; set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -72,7 +72,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -83,7 +83,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 37270771b72..378726f7c51 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1012,7 +1012,7 @@ CREATE DATABASE mydatabase IS_TEMPLATE = false OID = 966345; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1022,7 +1022,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1033,7 +1033,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1046,7 +1046,7 @@ WHERE datname = 'mydatabase'; \c - - - :master_port set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1057,7 +1057,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_1_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace @@ -1068,7 +1068,7 @@ WHERE datname = 'mydatabase'; \c - - - :worker_2_port -SELECT pd.datname , pd.datdba, pd.encoding, pd.datlocprovider, +SELECT pd.datname, pd.encoding, pd.datlocprovider, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace From 4c797889368e6e2d6897303c46e766476a12c53d Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 17:48:36 +0300 Subject: [PATCH 016/180] Fixes tablespace error --- src/test/regress/expected/create_drop_database_propagation.out | 2 +- src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 3321dd28996..71c72a08868 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -16,7 +16,7 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' - TABLESPACE = test_tablespace + TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; SELECT pd.datname, pd.encoding, diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 39344c7d8ba..edf8e7891f5 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -23,7 +23,7 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' - TABLESPACE = test_tablespace + TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; From aa47ca8bda94205b30d0f5804848572007ef13a5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 17:51:01 +0300 Subject: [PATCH 017/180] Fixes tablespace results --- .../expected/create_drop_database_propagation.out | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 71c72a08868..e578cd2ebdf 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -27,9 +27,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -41,9 +41,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -55,9 +55,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | test_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port From 29093e042a3b1756081b6ae398b23a24769f9db6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 19:44:57 +0300 Subject: [PATCH 018/180] Removes pg15 changes to test unusal failures --- src/test/regress/expected/pg15.out | 118 ----------------------------- src/test/regress/sql/pg15.sql | 112 --------------------------- 2 files changed, 230 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index c0e510cccd9..54cf7e81975 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1496,121 +1496,3 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; --- create/drop database for pg > 15 -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :master_port -create user create_drop_db_test_user; -set citus.enable_create_database_propagation=on; -CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'C' - LC_COLLATE = 'C' - LC_CTYPE = 'C' - ICU_LOCALE = 'C' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) - -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) - -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) - -\c - - - :master_port -set citus.enable_create_database_propagation=on; -drop database mydatabase; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- -(0 rows) - -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- -(0 rows) - -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- -(0 rows) - -\c - - - :master_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; -\c - - - :master_port -drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 378726f7c51..fe60222ddc4 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -976,115 +976,3 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; - - --- create/drop database for pg > 15 - - -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :master_port -create user create_drop_db_test_user; -set citus.enable_create_database_propagation=on; -CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'C' - LC_COLLATE = 'C' - LC_CTYPE = 'C' - ICU_LOCALE = 'C' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; - -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - - -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - - -\c - - - :master_port -set citus.enable_create_database_propagation=on; -drop database mydatabase; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - - -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - -\c - - - :worker_2_port - -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - -\c - - - :master_port -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; - -\c - - - :master_port -drop user create_drop_db_test_user; From 14618878c80cc7d5449c907e07966192af85bd04 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 20:35:22 +0300 Subject: [PATCH 019/180] Tests error cause --- .../create_drop_database_propagation.out | 27 +++++-------------- .../sql/create_drop_database_propagation.sql | 25 ----------------- 2 files changed, 6 insertions(+), 46 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index e578cd2ebdf..b2056d84a8d 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -1,11 +1,3 @@ -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; @@ -16,7 +8,6 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' - TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; SELECT pd.datname, pd.encoding, @@ -27,9 +18,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default (1 row) \c - - - :worker_1_port @@ -41,9 +32,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default (1 row) \c - - - :worker_2_port @@ -55,9 +46,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default (1 row) \c - - - :master_port @@ -101,11 +92,5 @@ WHERE datname = 'mydatabase'; --------------------------------------------------------------------- (0 rows) -\c - - - :master_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; \c - - - :master_port drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index edf8e7891f5..576274d6687 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -1,16 +1,3 @@ - - -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - \c - - - :master_port create user create_drop_db_test_user; @@ -23,7 +10,6 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' - TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; @@ -92,17 +78,6 @@ JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; -\c - - - :master_port - -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_1_port - -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_2_port - -drop tablespace create_drop_db_tablespace; \c - - - :master_port From c7304abd54680599ca25be3f119879e7ad311880 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 20:36:47 +0300 Subject: [PATCH 020/180] Fixes pg15 file --- src/test/regress/expected/pg15.out | 50 +++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 54cf7e81975..fcbb0cd1220 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -586,19 +586,59 @@ WHERE f.numeric_column = s.numeric_column; -- test new regex functions -- print order comments that contain the word `fluffily` at least twice SELECT o_comment FROM public.orders WHERE regexp_count(o_comment, 'FluFFily', 1, 'i')>=2 ORDER BY 1; -ERROR: relation "public.orders" does not exist + o_comment +--------------------------------------------------------------------- + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli + ly regular packages are fluffily even ideas. fluffily final + ng instructions integrate fluffily among the fluffily silent accounts. bli + ructions wake fluffily fluffily final gifts! furiou + s boost blithely fluffily idle ideas? fluffily even pin +(5 rows) + -- print the same items using a different regexp function SELECT o_comment FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1; -ERROR: relation "public.orders" does not exist + o_comment +--------------------------------------------------------------------- + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli + ly regular packages are fluffily even ideas. fluffily final + ng instructions integrate fluffily among the fluffily silent accounts. bli + ructions wake fluffily fluffily final gifts! furiou + s boost blithely fluffily idle ideas? fluffily even pin +(5 rows) + -- print the position where we find the second fluffily in the comment SELECT o_comment, regexp_instr(o_comment, 'fluffily.*(fluffily)') FROM public.orders ORDER BY 2 desc LIMIT 5; -ERROR: relation "public.orders" does not exist + o_comment | regexp_instr +--------------------------------------------------------------------- + ng instructions integrate fluffily among the fluffily silent accounts. bli | 27 + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli | 26 + ly regular packages are fluffily even ideas. fluffily final | 25 + s boost blithely fluffily idle ideas? fluffily even pin | 18 + ructions wake fluffily fluffily final gifts! furiou | 15 +(5 rows) + -- print the substrings between two `fluffily` SELECT regexp_substr(o_comment, 'fluffily.*fluffily') FROM public.orders ORDER BY 1 LIMIT 5; -ERROR: relation "public.orders" does not exist + regexp_substr +--------------------------------------------------------------------- + fluffily among the fluffily + fluffily even ideas. fluffily + fluffily fluffily + fluffily fluffily + fluffily idle ideas? fluffily +(5 rows) + -- replace second `fluffily` with `silkily` SELECT regexp_replace(o_comment, 'fluffily', 'silkily', 1, 2) FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1 desc; -ERROR: relation "public.orders" does not exist + regexp_replace +--------------------------------------------------------------------- + s boost blithely fluffily idle ideas? silkily even pin + ructions wake fluffily silkily final gifts! furiou + ng instructions integrate fluffily among the silkily silent accounts. bli + ly regular packages are fluffily even ideas. silkily final + al, bold deposits cajole fluffily silkily final foxes. pending ideas beli +(5 rows) + -- test new COPY features -- COPY TO statements with text format and headers CREATE TABLE copy_test(id int, data int); From acece9818ce08709c51fc8bea4b8e56027cf1a04 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 20:49:19 +0300 Subject: [PATCH 021/180] Rollbacks pg_regress_multi --- src/test/regress/pg_regress_multi.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index ba2251e37e2..4cc022198c4 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -630,7 +630,7 @@ sub generate_hba } } -for my $tablespace ("ts0", "ts1", "ts2", "ts3", "ts4", "ts5") +for my $tablespace ("ts0", "ts1", "ts2") { if (-e catfile($TMP_CHECKDIR, $tablespace)) { From 5eb87839b678c6877ee3d0d3d72bb580475b345e Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 21:34:14 +0300 Subject: [PATCH 022/180] Rollbacks last test --- src/test/regress/multi_1_schedule | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 6cd6225ef42..ad70f136e5a 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -52,7 +52,6 @@ test: multi_read_from_secondaries test: grant_on_database_propagation test: alter_database_propagation -test: create_drop_database_propagation test: citus_shards From 1d9c02fab902371c956526dee99e000be76dbc1d Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 21:48:28 +0300 Subject: [PATCH 023/180] Changes schedule location --- src/test/regress/multi_1_schedule | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index ad70f136e5a..fee725307ee 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -15,6 +15,7 @@ # --- # Tests around schema changes, these are run first, so there's no preexisting objects. # --- +test: create_drop_database_propagation test: multi_extension test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw test: single_node From c3d773ac6f4ea89d8f997343dfae661de0d209f6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 22:14:12 +0300 Subject: [PATCH 024/180] Changes test order --- src/test/regress/multi_1_schedule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index fee725307ee..6cd6225ef42 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -15,7 +15,6 @@ # --- # Tests around schema changes, these are run first, so there's no preexisting objects. # --- -test: create_drop_database_propagation test: multi_extension test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw test: single_node @@ -53,6 +52,7 @@ test: multi_read_from_secondaries test: grant_on_database_propagation test: alter_database_propagation +test: create_drop_database_propagation test: citus_shards From 51d1560b349c1372316a288ddecf486f4b517bc8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 5 Oct 2023 22:30:06 +0300 Subject: [PATCH 025/180] Adds OID to see the error cause --- src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 576274d6687..c86523e37c7 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -13,7 +13,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.datname, pd.encoding, +SELECT pd.oid,pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace From 1f578791bbe628fad4c66421ce65c2ba342d9ff9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 15:18:16 +0300 Subject: [PATCH 026/180] Fixes pg_dist_object delete problem --- src/backend/distributed/commands/database.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index c7da5d06889..fc91ae8c3aa 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -380,9 +380,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) char *command = text_to_cstring(commandText); Node *parseTree = ParseTreeNode(command); - ereport(NOTICE, (errmsg("test internal pre"), - errhint("test pre hint"))); - set_config_option("citus.enable_ddl_propagation", "off", (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); @@ -411,7 +408,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) { DropdbStmt *stmt = castNode(DropdbStmt, parseTree); - bool missingOk = true; + bool missingOk = false; Oid databaseOid = get_database_oid(stmt->dbname, missingOk); if (!OidIsValid(databaseOid)) @@ -420,14 +417,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) } else { - /* remove database from pg_dist_object */ - ObjectAddress dbAddress = { 0 }; - ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); - - if (IsObjectDistributed(&dbAddress)) - { - UnmarkObjectDistributed(&dbAddress); - } /* / * remove database from database shards * / */ /* DeleteDatabaseShardByDatabaseIdLocally(databaseOid); */ @@ -486,7 +475,15 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, "SELECT pg_catalog.citus_internal_database_command(%s)", quote_literal_cstr(dropDatabaseCommand)); + /* Delete from pg_dist_object */ + + if (IsObjectDistributed(&dbAddress)) + { + UnmarkObjectDistributed(&dbAddress); + } + /* we execute here to avoid EnsureCoordinator check in ExecuteDistributedDDLJob */ + bool outsideTransaction = false; List *taskList = CreateDDLTaskList(internalDropCommand->data, workerNodes, outsideTransaction); From 94abb074090299cc3bfb0f73150aa7fad67e8062 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 15:22:15 +0300 Subject: [PATCH 027/180] Fixes indentation --- src/backend/distributed/commands/database.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index fc91ae8c3aa..a12ff3f127e 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -417,7 +417,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) } else { - /* / * remove database from database shards * / */ /* DeleteDatabaseShardByDatabaseIdLocally(databaseOid); */ From 6913496f42ada59742808afbc63f9bbae1c21b68 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 15:24:23 +0300 Subject: [PATCH 028/180] Fixes tests --- src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index c86523e37c7..576274d6687 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -13,7 +13,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.oid,pd.datname, pd.encoding, +SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace From a7112fb9651f01846886d5cae2d8312b3b94b86a Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 15:45:05 +0300 Subject: [PATCH 029/180] Adds tablespace to test tablespace statement --- .../create_drop_database_propagation.out | 27 ++++++++++++++----- src/test/regress/pg_regress_multi.pl | 2 +- .../sql/create_drop_database_propagation.sql | 27 +++++++++++++++++-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index b2056d84a8d..e578cd2ebdf 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -1,3 +1,11 @@ +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; @@ -8,6 +16,7 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' + TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; SELECT pd.datname, pd.encoding, @@ -18,9 +27,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -32,9 +41,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -46,9 +55,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | pg_default + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port @@ -92,5 +101,11 @@ WHERE datname = 'mydatabase'; --------------------------------------------------------------------- (0 rows) +\c - - - :master_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; \c - - - :master_port drop user create_drop_db_test_user; diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 4cc022198c4..ba2251e37e2 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -630,7 +630,7 @@ sub generate_hba } } -for my $tablespace ("ts0", "ts1", "ts2") +for my $tablespace ("ts0", "ts1", "ts2", "ts3", "ts4", "ts5") { if (-e catfile($TMP_CHECKDIR, $tablespace)) { diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 576274d6687..415a03f9743 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -1,3 +1,16 @@ + + +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + \c - - - :master_port create user create_drop_db_test_user; @@ -10,6 +23,7 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' LC_COLLATE = 'C' LC_CTYPE = 'C' + TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; @@ -78,9 +92,18 @@ JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - \c - - - :master_port -drop user create_drop_db_test_user; +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port + +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; + +\c - - - :master_port + +drop user create_drop_db_test_user; From 6f3655672412243d7634336d6ea4348d8f5b33e3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 15:47:42 +0300 Subject: [PATCH 030/180] Revert "Removes pg15 changes to test unusal failures" This reverts commit 85963e9cb5f1f9dd405d0c7cf2b8f8de4100066a. --- src/test/regress/expected/pg15.out | 118 +++++++++++++++++++++++++++++ src/test/regress/sql/pg15.sql | 112 +++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index fcbb0cd1220..6165990fe73 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1536,3 +1536,121 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; +-- create/drop database for pg > 15 +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :worker_1_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :worker_2_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- + mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace +(1 row) + +\c - - - :master_port +set citus.enable_create_database_propagation=on; +drop database mydatabase; +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_1_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :worker_2_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +--------------------------------------------------------------------- +(0 rows) + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; +\c - - - :master_port +drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index fe60222ddc4..378726f7c51 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -976,3 +976,115 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; + + +-- create/drop database for pg > 15 + + +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'C' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + ICU_LOCALE = 'C' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :worker_1_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :worker_2_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :master_port +set citus.enable_create_database_propagation=on; +drop database mydatabase; +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + + +\c - - - :worker_1_port +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :worker_2_port + +SELECT pd.datname, pd.encoding, pd.datlocprovider, +pd.datistemplate, pd.datallowconn, pd.datconnlimit, +pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, +pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace +FROM pg_database pd +JOIN pg_authid pa ON pd.datdba = pa.oid +join pg_tablespace pt on pd.dattablespace = pt.oid +WHERE datname = 'mydatabase'; + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; + +\c - - - :master_port +drop user create_drop_db_test_user; From 214ebc4374dccc779b68cc9997a79b624ed4e363 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 16:10:47 +0300 Subject: [PATCH 031/180] Fixes encodings for pg16 --- src/test/regress/expected/pg15.out | 70 +++++++----------------------- src/test/regress/sql/pg15.sql | 8 ++-- 2 files changed, 19 insertions(+), 59 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 6165990fe73..2fa43788b39 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -586,59 +586,19 @@ WHERE f.numeric_column = s.numeric_column; -- test new regex functions -- print order comments that contain the word `fluffily` at least twice SELECT o_comment FROM public.orders WHERE regexp_count(o_comment, 'FluFFily', 1, 'i')>=2 ORDER BY 1; - o_comment ---------------------------------------------------------------------- - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli - ly regular packages are fluffily even ideas. fluffily final - ng instructions integrate fluffily among the fluffily silent accounts. bli - ructions wake fluffily fluffily final gifts! furiou - s boost blithely fluffily idle ideas? fluffily even pin -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the same items using a different regexp function SELECT o_comment FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1; - o_comment ---------------------------------------------------------------------- - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli - ly regular packages are fluffily even ideas. fluffily final - ng instructions integrate fluffily among the fluffily silent accounts. bli - ructions wake fluffily fluffily final gifts! furiou - s boost blithely fluffily idle ideas? fluffily even pin -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the position where we find the second fluffily in the comment SELECT o_comment, regexp_instr(o_comment, 'fluffily.*(fluffily)') FROM public.orders ORDER BY 2 desc LIMIT 5; - o_comment | regexp_instr ---------------------------------------------------------------------- - ng instructions integrate fluffily among the fluffily silent accounts. bli | 27 - al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli | 26 - ly regular packages are fluffily even ideas. fluffily final | 25 - s boost blithely fluffily idle ideas? fluffily even pin | 18 - ructions wake fluffily fluffily final gifts! furiou | 15 -(5 rows) - +ERROR: relation "public.orders" does not exist -- print the substrings between two `fluffily` SELECT regexp_substr(o_comment, 'fluffily.*fluffily') FROM public.orders ORDER BY 1 LIMIT 5; - regexp_substr ---------------------------------------------------------------------- - fluffily among the fluffily - fluffily even ideas. fluffily - fluffily fluffily - fluffily fluffily - fluffily idle ideas? fluffily -(5 rows) - +ERROR: relation "public.orders" does not exist -- replace second `fluffily` with `silkily` SELECT regexp_replace(o_comment, 'fluffily', 'silkily', 1, 2) FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1 desc; - regexp_replace ---------------------------------------------------------------------- - s boost blithely fluffily idle ideas? silkily even pin - ructions wake fluffily silkily final gifts! furiou - ng instructions integrate fluffily among the silkily silent accounts. bli - ly regular packages are fluffily even ideas. silkily final - al, bold deposits cajole fluffily silkily final foxes. pending ideas beli -(5 rows) - +ERROR: relation "public.orders" does not exist -- test new COPY features -- COPY TO statements with text format and headers CREATE TABLE copy_test(id int, data int); @@ -1554,10 +1514,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'C' - LC_COLLATE = 'C' - LC_CTYPE = 'C' - ICU_LOCALE = 'C' + LOCALE = 'en_US.UTF-8' + LC_COLLATE = 'en_US.UTF-8' + LC_CTYPE = 'en_US.UTF-8' + ICU_LOCALE = 'en_US.UTF-8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace @@ -1572,9 +1532,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1586,9 +1546,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1600,9 +1560,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | C | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 378726f7c51..86afdea2b87 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1001,10 +1001,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'C' - LC_COLLATE = 'C' - LC_CTYPE = 'C' - ICU_LOCALE = 'C' + LOCALE = 'en_US.UTF-8' + LC_COLLATE = 'en_US.UTF-8' + LC_CTYPE = 'en_US.UTF-8' + ICU_LOCALE = 'en_US.UTF-8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace From 3050b0eab9337868df75ade7c7bbcf7c636b909f Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 16:41:07 +0300 Subject: [PATCH 032/180] Fixes pg15 tests --- src/test/regress/expected/pg15.out | 50 +++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 2fa43788b39..13a3bcd7bec 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -586,19 +586,59 @@ WHERE f.numeric_column = s.numeric_column; -- test new regex functions -- print order comments that contain the word `fluffily` at least twice SELECT o_comment FROM public.orders WHERE regexp_count(o_comment, 'FluFFily', 1, 'i')>=2 ORDER BY 1; -ERROR: relation "public.orders" does not exist + o_comment +--------------------------------------------------------------------- + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli + ly regular packages are fluffily even ideas. fluffily final + ng instructions integrate fluffily among the fluffily silent accounts. bli + ructions wake fluffily fluffily final gifts! furiou + s boost blithely fluffily idle ideas? fluffily even pin +(5 rows) + -- print the same items using a different regexp function SELECT o_comment FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1; -ERROR: relation "public.orders" does not exist + o_comment +--------------------------------------------------------------------- + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli + ly regular packages are fluffily even ideas. fluffily final + ng instructions integrate fluffily among the fluffily silent accounts. bli + ructions wake fluffily fluffily final gifts! furiou + s boost blithely fluffily idle ideas? fluffily even pin +(5 rows) + -- print the position where we find the second fluffily in the comment SELECT o_comment, regexp_instr(o_comment, 'fluffily.*(fluffily)') FROM public.orders ORDER BY 2 desc LIMIT 5; -ERROR: relation "public.orders" does not exist + o_comment | regexp_instr +--------------------------------------------------------------------- + ng instructions integrate fluffily among the fluffily silent accounts. bli | 27 + al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli | 26 + ly regular packages are fluffily even ideas. fluffily final | 25 + s boost blithely fluffily idle ideas? fluffily even pin | 18 + ructions wake fluffily fluffily final gifts! furiou | 15 +(5 rows) + -- print the substrings between two `fluffily` SELECT regexp_substr(o_comment, 'fluffily.*fluffily') FROM public.orders ORDER BY 1 LIMIT 5; -ERROR: relation "public.orders" does not exist + regexp_substr +--------------------------------------------------------------------- + fluffily among the fluffily + fluffily even ideas. fluffily + fluffily fluffily + fluffily fluffily + fluffily idle ideas? fluffily +(5 rows) + -- replace second `fluffily` with `silkily` SELECT regexp_replace(o_comment, 'fluffily', 'silkily', 1, 2) FROM public.orders WHERE regexp_like(o_comment, 'fluffily.*fluffily') ORDER BY 1 desc; -ERROR: relation "public.orders" does not exist + regexp_replace +--------------------------------------------------------------------- + s boost blithely fluffily idle ideas? silkily even pin + ructions wake fluffily silkily final gifts! furiou + ng instructions integrate fluffily among the silkily silent accounts. bli + ly regular packages are fluffily even ideas. silkily final + al, bold deposits cajole fluffily silkily final foxes. pending ideas beli +(5 rows) + -- test new COPY features -- COPY TO statements with text format and headers CREATE TABLE copy_test(id int, data int); From 576264ceca5f9cee0825f8d2a5523004592a888b Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 17:01:27 +0300 Subject: [PATCH 033/180] Fixes UTF error --- src/test/regress/expected/pg15.out | 14 +++++++------- src/test/regress/sql/pg15.sql | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 13a3bcd7bec..57a095c12f3 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1554,10 +1554,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.UTF-8' - LC_COLLATE = 'en_US.UTF-8' - LC_CTYPE = 'en_US.UTF-8' - ICU_LOCALE = 'en_US.UTF-8' + LOCALE = 'en_US.utf-8' + LC_COLLATE = 'en_US.utf-8' + LC_CTYPE = 'en_US.utf-8' + ICU_LOCALE = 'en_US.utf-8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace @@ -1574,7 +1574,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1588,7 +1588,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1602,7 +1602,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.UTF-8 | en_US.UTF-8 | en_US.UTF-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 86afdea2b87..ac5f6eec168 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1001,10 +1001,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.UTF-8' - LC_COLLATE = 'en_US.UTF-8' - LC_CTYPE = 'en_US.UTF-8' - ICU_LOCALE = 'en_US.UTF-8' + LOCALE = 'en_US.utf-8' + LC_COLLATE = 'en_US.utf-8' + LC_CTYPE = 'en_US.utf-8' + ICU_LOCALE = 'en_US.utf-8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace From 031b6480f2460c61d1ba68799b9bcb704b0a5331 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 17:03:57 +0300 Subject: [PATCH 034/180] Fixes locale --- src/test/regress/expected/pg15.out | 14 +++++++------- src/test/regress/sql/pg15.sql | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 57a095c12f3..338e33c5c38 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1554,10 +1554,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.utf-8' - LC_COLLATE = 'en_US.utf-8' - LC_CTYPE = 'en_US.utf-8' - ICU_LOCALE = 'en_US.utf-8' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'en_US.utf8' + LC_CTYPE = 'en_US.utf8' + ICU_LOCALE = 'en_US.utf8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace @@ -1574,7 +1574,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1588,7 +1588,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1602,7 +1602,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf-8 | en_US.utf-8 | en_US.utf-8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index ac5f6eec168..1c1936dfce2 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1001,10 +1001,10 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.utf-8' - LC_COLLATE = 'en_US.utf-8' - LC_CTYPE = 'en_US.utf-8' - ICU_LOCALE = 'en_US.utf-8' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'en_US.utf8' + LC_CTYPE = 'en_US.utf8' + ICU_LOCALE = 'en_US.utf8' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace From da46878a8f334c58db93956edf7db94a7f4b48f4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 17:50:10 +0300 Subject: [PATCH 035/180] Fixes Locale --- src/test/regress/expected/pg15.out | 16 ++++++++-------- src/test/regress/sql/pg15.sql | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 338e33c5c38..e210ac6e391 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1554,7 +1554,7 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' + LOCALE = '' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8' ICU_LOCALE = 'en_US.utf8' @@ -1572,9 +1572,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1586,9 +1586,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1600,9 +1600,9 @@ FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port @@ -1653,4 +1653,4 @@ drop tablespace create_drop_db_tablespace; \c - - - :worker_2_port drop tablespace create_drop_db_tablespace; \c - - - :master_port -drop user create_drop_db_test_user; +drop user create_drop_db_test_user; \ No newline at end of file diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 1c1936dfce2..3fc4cf0b211 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1001,7 +1001,7 @@ CREATE DATABASE mydatabase CONNECTION LIMIT = 10 ENCODING = 'UTF8' STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' + LOCALE = '' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8' ICU_LOCALE = 'en_US.utf8' From ddea1c897b25fdb09c26b43510590e81c460e86e Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 18:02:43 +0300 Subject: [PATCH 036/180] Removes iculocale --- src/test/regress/expected/pg15.out | 4 ++-- src/test/regress/sql/pg15.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index e210ac6e391..76a89706e6a 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1557,7 +1557,7 @@ CREATE DATABASE mydatabase LOCALE = '' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8' - ICU_LOCALE = 'en_US.utf8' + ICU_LOCALE = '' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace @@ -1574,7 +1574,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 3fc4cf0b211..32ced075082 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1004,7 +1004,7 @@ CREATE DATABASE mydatabase LOCALE = '' LC_COLLATE = 'en_US.utf8' LC_CTYPE = 'en_US.utf8' - ICU_LOCALE = 'en_US.utf8' + ICU_LOCALE = '' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace From 32d74d95218211708cb58d14d19fc638b11eeb4d Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 18:14:00 +0300 Subject: [PATCH 037/180] Fixes collate and ctype --- src/test/regress/expected/pg15.out | 4 ++-- src/test/regress/sql/pg15.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 76a89706e6a..739985d4fd9 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1555,8 +1555,8 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' STRATEGY = 'wal_log' LOCALE = '' - LC_COLLATE = 'en_US.utf8' - LC_CTYPE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' ICU_LOCALE = '' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 32ced075082..ddc02b2e4c3 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1002,8 +1002,8 @@ CREATE DATABASE mydatabase ENCODING = 'UTF8' STRATEGY = 'wal_log' LOCALE = '' - LC_COLLATE = 'en_US.utf8' - LC_CTYPE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' ICU_LOCALE = '' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' From d55ecc23ae175072a84f283e7565fa1289473fd5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 18:33:10 +0300 Subject: [PATCH 038/180] Updates test results --- src/test/regress/expected/pg15.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 739985d4fd9..6cf7a04a537 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1557,7 +1557,7 @@ CREATE DATABASE mydatabase LOCALE = '' LC_COLLATE = 'POSIX' LC_CTYPE = 'POSIX' - ICU_LOCALE = '' + ICU_LOCALE = 'und' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace @@ -1574,7 +1574,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1588,7 +1588,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1602,7 +1602,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | en_US.utf8 | en_US.utf8 | en_US.utf8 | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port From 7bccf9318327272ffaee349cb3ab6105f9cf1c86 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 18:58:52 +0300 Subject: [PATCH 039/180] Fixes icu locale --- src/test/regress/sql/pg15.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index ddc02b2e4c3..f0c6706d2b2 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1004,7 +1004,7 @@ CREATE DATABASE mydatabase LOCALE = '' LC_COLLATE = 'POSIX' LC_CTYPE = 'POSIX' - ICU_LOCALE = '' + ICU_LOCALE = 'und' LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace From 09d201d1a8f57fcb022355171906b3b008deacff Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 6 Oct 2023 19:14:21 +0300 Subject: [PATCH 040/180] Fixes pg15 output --- src/test/regress/expected/pg15.out | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 6cf7a04a537..caee521a745 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1574,7 +1574,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port @@ -1588,7 +1588,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_2_port @@ -1602,7 +1602,7 @@ join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :master_port From 0bf1c2c139afc02bf3ee7076d152879f536f6f84 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 13:27:12 +0300 Subject: [PATCH 041/180] Refactors the code to make more managable --- .../distributed/deparser/citus_deparseutils.c | 56 +++++++ .../deparser/deparse_database_stmts.c | 139 +++--------------- src/include/distributed/deparser.h | 14 ++ 3 files changed, 89 insertions(+), 120 deletions(-) create mode 100644 src/backend/distributed/deparser/citus_deparseutils.c diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c new file mode 100644 index 00000000000..b2798d3e1ca --- /dev/null +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -0,0 +1,56 @@ +#include "postgres.h" +#include "utils/builtins.h" +#include "commands/defrem.h" +#include "utils/elog.h" +#include "utils/rel.h" +#include "utils/relcache.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "distributed/deparser.h" + + + +void +handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_formats, int + opt_formats_len) +{ + const char *name = option->defname; + int i; + + for (i = 0; i < opt_formats_len; i++) + { + if (strcmp(name, opt_formats[i].name) == 0) + { + switch (opt_formats[i].type) + { + case T_String: + { + char *value = defGetString(option); + appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); + break; + } + + case T_Integer: + { + int32 value = defGetInt32(option); + appendStringInfo(buf, opt_formats[i].format, value); + break; + } + + case T_Boolean: + { + bool value = defGetBoolean(option); + appendStringInfo(buf, opt_formats[i].format, value ? "true" : + "false"); + break; + } + + default: + + /* Should not happen */ + elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); + } + return; + } + } +} diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 6ffce3b4832..6ae0a0e59ee 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -207,8 +207,24 @@ DeparseAlterDatabaseSetStmt(Node *node) } +const struct option_format option_formats[] = { + { "template", " TEMPLATE %s", T_String }, + { "owner", " OWNER %s", T_String }, + { "tablespace", " TABLESPACE %s", T_String }, + { "connection_limit", " CONNECTION LIMIT %d", T_Integer }, + { "encoding", " ENCODING %s", T_String }, + { "lc_collate", " LC_COLLATE %s", T_String }, + { "lc_ctype", " LC_CTYPE %s", T_String }, + { "icu_locale", " ICU_LOCALE %s", T_String }, + { "icu_rules", " ICU_RULES %s", T_String }, + { "locale_provider", " LOCALE_PROVIDER %s", T_String }, + { "is_template", " IS_TEMPLATE %s", T_Boolean }, + { "allow_connections", " ALLOW_CONNECTIONS %s", T_Boolean }, +}; + + static void -AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) +AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { appendStringInfo(buf, "CREATE DATABASE %s", @@ -218,124 +234,7 @@ AppendCreatedbStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - if (strcmp(option->defname, "tablespace") == 0) - { - char *tablespaceName = defGetString(option); - - appendStringInfo(buf, " TABLESPACE %s", - quote_identifier(tablespaceName)); - } - else if (strcmp(option->defname, "owner") == 0) - { - char *owner = defGetString(option); - - appendStringInfo(buf, " OWNER %s", - quote_identifier(owner)); - } - else if (strcmp(option->defname, "template") == 0) - { - char *template = defGetString(option); - - appendStringInfo(buf, " TEMPLATE %s", - quote_identifier(template)); - } - else if (strcmp(option->defname, "encoding") == 0) - { - char *encoding = defGetString(option); - - appendStringInfo(buf, " ENCODING %s", - quote_literal_cstr(encoding)); - } - else if (strcmp(option->defname, "locale") == 0) - { - char *locale = defGetString(option); - - appendStringInfo(buf, " LOCALE %s", - quote_literal_cstr(locale)); - } - else if (strcmp(option->defname, "lc_collate") == 0) - { - char *lc_collate = defGetString(option); - - appendStringInfo(buf, " LC_COLLATE %s", - quote_literal_cstr(lc_collate)); - } - else if (strcmp(option->defname, "lc_ctype") == 0) - { - char *lc_ctype = defGetString(option); - - appendStringInfo(buf, " LC_CTYPE %s", - quote_literal_cstr(lc_ctype)); - } - else if (strcmp(option->defname, "icu_locale") == 0) - { - char *icuLocale = defGetString(option); - - appendStringInfo(buf, " ICU_LOCALE %s", - quote_literal_cstr(icuLocale)); - } - else if (strcmp(option->defname, "icu_rules") == 0) - { - char *icuLocale = defGetString(option); - - appendStringInfo(buf, " ICU_RULES %s", - quote_literal_cstr(icuLocale)); - } - else if (strcmp(option->defname, "locale_provider") == 0) - { - char *localeProvider = defGetString(option); - - appendStringInfo(buf, " LOCALE_PROVIDER %s", - quote_literal_cstr(localeProvider)); - } - else if (strcmp(option->defname, "is_template") == 0) - { - bool isTemplate = defGetBoolean(option); - - appendStringInfo(buf, " IS_TEMPLATE %s", - isTemplate ? "true" : "false"); - } - else if (strcmp(option->defname, "allow_connections") == 0) - { - bool allowConnections = defGetBoolean(option); - - appendStringInfo(buf, " ALLOW_CONNECTIONS %s", - allowConnections ? "true" : "false"); - } - else if (strcmp(option->defname, "connection_limit") == 0) - { - int connectionLimit = defGetInt32(option); - - appendStringInfo(buf, " CONNECTION_LIMIT %d", connectionLimit); - } -#if PG_VERSION_NUM >= PG_VERSION_15 - else if (strcmp(option->defname, "collation_version") == 0) - { - char *collationVersion = defGetString(option); - - appendStringInfo(buf, " COLLATION_VERSION %s", - quote_literal_cstr(collationVersion)); - } - else if (strcmp(option->defname, "oid") == 0) - { - Oid objectId = defGetObjectId(option); - - appendStringInfo(buf, " OID %d", objectId); - } - else if (strcmp(option->defname, "strategy") == 0) - { - char *strategy = defGetString(option); - - appendStringInfo(buf, " STRATEGY %s", - quote_literal_cstr(strategy)); - } -#endif - else - { - ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized CREATE DATABASE option \"%s\"", - option->defname))); - } + handleOption(buf, option, option_formats, lengthof(option_formats)); } } @@ -347,7 +246,7 @@ DeparseCreateDatabaseStmt(Node *node) StringInfoData str = { 0 }; initStringInfo(&str); - AppendCreatedbStmt(&str, stmt); + AppendCreateDatabaseStmt(&str, stmt); return str.data; } diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 6d76b03dd7c..b5528797806 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -121,6 +121,20 @@ extern void AppendGrantedByInGrant(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedPrefix(StringInfo buf, GrantStmt *stmt); extern void AppendGrantSharedSuffix(StringInfo buf, GrantStmt *stmt); +/* Common deparser utils */ + +struct option_format +{ + const char *name; + const char *format; + int type; +}; + + +extern void handleOption(StringInfo buf, DefElem *option, const struct + option_format *opt_formats, int + opt_formats_len); + /* forward declarations for deparse_statistics_stmts.c */ extern char * DeparseCreateStatisticsStmt(Node *node); From 82358fb37d9d7d859c9d67e7491194745c2058b5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 13:28:23 +0300 Subject: [PATCH 042/180] Fixes indentation --- src/backend/distributed/deparser/citus_deparseutils.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index b2798d3e1ca..9b694e7267f 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -9,7 +9,6 @@ #include "distributed/deparser.h" - void handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_formats, int opt_formats_len) From e7f80580164ce3dab5c40e243256e2b9c31a9d4f Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 13:46:54 +0300 Subject: [PATCH 043/180] Removes T_Boolean since it breaks pg14 compile --- .../distributed/deparser/citus_deparseutils.c | 46 ++++++++----------- .../deparser/deparse_database_stmts.c | 24 +++++----- src/include/distributed/deparser.h | 2 +- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 9b694e7267f..9e65f5700c8 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -20,36 +20,26 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo { if (strcmp(name, opt_formats[i].name) == 0) { - switch (opt_formats[i].type) + if (strcmp(opt_formats[i].type, "string") == 0) { - case T_String: - { - char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); - break; - } - - case T_Integer: - { - int32 value = defGetInt32(option); - appendStringInfo(buf, opt_formats[i].format, value); - break; - } - - case T_Boolean: - { - bool value = defGetBoolean(option); - appendStringInfo(buf, opt_formats[i].format, value ? "true" : - "false"); - break; - } - - default: - - /* Should not happen */ - elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); + char *value = defGetString(option); + appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); + } + else if (strcmp(opt_formats[i].type, "integer") == 0) + { + int32 value = defGetInt32(option); + appendStringInfo(buf, opt_formats[i].format, value); + } + else if (strcmp(opt_formats[i].type, "boolean") == 0) + { + bool value = defGetBoolean(option); + appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); + } + else + { + elog(ERROR, "unrecognized option type: %s", opt_formats[i].type); } - return; + break; } } } diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 6ae0a0e59ee..094c6fc1e72 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -208,18 +208,18 @@ DeparseAlterDatabaseSetStmt(Node *node) const struct option_format option_formats[] = { - { "template", " TEMPLATE %s", T_String }, - { "owner", " OWNER %s", T_String }, - { "tablespace", " TABLESPACE %s", T_String }, - { "connection_limit", " CONNECTION LIMIT %d", T_Integer }, - { "encoding", " ENCODING %s", T_String }, - { "lc_collate", " LC_COLLATE %s", T_String }, - { "lc_ctype", " LC_CTYPE %s", T_String }, - { "icu_locale", " ICU_LOCALE %s", T_String }, - { "icu_rules", " ICU_RULES %s", T_String }, - { "locale_provider", " LOCALE_PROVIDER %s", T_String }, - { "is_template", " IS_TEMPLATE %s", T_Boolean }, - { "allow_connections", " ALLOW_CONNECTIONS %s", T_Boolean }, + { "template", " TEMPLATE %s", "string" }, + { "owner", " OWNER %s", "string" }, + { "tablespace", " TABLESPACE %s", "string" }, + { "connection_limit", " CONNECTION LIMIT %d", "integer" }, + { "encoding", " ENCODING %s", "string" }, + { "lc_collate", " LC_COLLATE %s", "string" }, + { "lc_ctype", " LC_CTYPE %s", "string" }, + { "icu_locale", " ICU_LOCALE %s", "string" }, + { "icu_rules", " ICU_RULES %s", "string" }, + { "locale_provider", " LOCALE_PROVIDER %s", "string" }, + { "is_template", " IS_TEMPLATE %s", "boolean" }, + { "allow_connections", " ALLOW_CONNECTIONS %s", "boolean" }, }; diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index b5528797806..428c91a5f06 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -127,7 +127,7 @@ struct option_format { const char *name; const char *format; - int type; + const char *type; }; From 6096a0ed5744fd2c96aac5b8af4abf63339eac95 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 15:14:13 +0300 Subject: [PATCH 044/180] Adds missing types --- .../distributed/deparser/citus_deparseutils.c | 10 +++++ .../deparser/deparse_database_stmts.c | 38 ++++++++++--------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 9e65f5700c8..546360fdfc0 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -35,6 +35,16 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo bool value = defGetBoolean(option); appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); } + else if (strcmp(opt_formats[i].type, "object_id") == 0) + { + Oid value = defGetObjectId(option); + appendStringInfo(buf, opt_formats[i].format, value ); + } + else if (strcmp(opt_formats[i].type, "literal_cstr") == 0) + { + char *value = defGetString(option); + appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value) ); + } else { elog(ERROR, "unrecognized option type: %s", opt_formats[i].type); diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 094c6fc1e72..bc5d48f363e 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -30,6 +30,25 @@ static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendDefElemConnLimit(StringInfo buf, DefElem *def); +const struct option_format create_database_option_formats[] = { + { "template", " TEMPLATE %s", "string" }, + { "owner", " OWNER %s", "string" }, + { "tablespace", " TABLESPACE %s", "string" }, + { "connection_limit", " CONNECTION LIMIT %d", "integer" }, + { "encoding", " ENCODING %s", "literal_cstr" }, + { "locale", " LOCALE %s", "literal_cstr" }, + { "lc_collate", " LC_COLLATE %s", "literal_cstr" }, + { "lc_ctype", " LC_CTYPE %s", "literal_cstr" }, + { "icu_locale", " ICU_LOCALE %s", "literal_cstr" }, + { "icu_rules", " ICU_RULES %s", "literal_cstr" }, + { "locale_provider", " LOCALE_PROVIDER %s", "literal_cstr" }, + { "is_template", " IS_TEMPLATE %s", "boolean" }, + { "allow_connections", " ALLOW_CONNECTIONS %s", "boolean" }, + { "collation_version", " COLLATION_VERSION %s", "literal_cstr" }, + { "strategy", " STRATEGY %s", "literal_cstr" }, + { "oid", " OID %d", "object_id" }, +}; + char * DeparseAlterDatabaseOwnerStmt(Node *node) { @@ -207,22 +226,6 @@ DeparseAlterDatabaseSetStmt(Node *node) } -const struct option_format option_formats[] = { - { "template", " TEMPLATE %s", "string" }, - { "owner", " OWNER %s", "string" }, - { "tablespace", " TABLESPACE %s", "string" }, - { "connection_limit", " CONNECTION LIMIT %d", "integer" }, - { "encoding", " ENCODING %s", "string" }, - { "lc_collate", " LC_COLLATE %s", "string" }, - { "lc_ctype", " LC_CTYPE %s", "string" }, - { "icu_locale", " ICU_LOCALE %s", "string" }, - { "icu_rules", " ICU_RULES %s", "string" }, - { "locale_provider", " LOCALE_PROVIDER %s", "string" }, - { "is_template", " IS_TEMPLATE %s", "boolean" }, - { "allow_connections", " ALLOW_CONNECTIONS %s", "boolean" }, -}; - - static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -234,7 +237,8 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - handleOption(buf, option, option_formats, lengthof(option_formats)); + handleOption(buf, option, create_database_option_formats, lengthof( + create_database_option_formats)); } } From 134ec8f4284380172f4814d426517e211d6a0894 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 15:56:05 +0300 Subject: [PATCH 045/180] Fixes compile error for PG 14 --- src/backend/distributed/deparser/citus_deparseutils.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 546360fdfc0..d7554aa79e6 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -35,11 +35,13 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo bool value = defGetBoolean(option); appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); } +#if PG_VERSION_NUM >= PG_VERSION_15 else if (strcmp(opt_formats[i].type, "object_id") == 0) { Oid value = defGetObjectId(option); appendStringInfo(buf, opt_formats[i].format, value ); } +#endif else if (strcmp(opt_formats[i].type, "literal_cstr") == 0) { char *value = defGetString(option); From df715a0402a4bcc45e18a003602f03d475ff97b7 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 15:57:34 +0300 Subject: [PATCH 046/180] Fixes indent --- src/backend/distributed/deparser/citus_deparseutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index d7554aa79e6..692d10e1ce6 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -39,13 +39,13 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo else if (strcmp(opt_formats[i].type, "object_id") == 0) { Oid value = defGetObjectId(option); - appendStringInfo(buf, opt_formats[i].format, value ); + appendStringInfo(buf, opt_formats[i].format, value); } #endif else if (strcmp(opt_formats[i].type, "literal_cstr") == 0) { char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value) ); + appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value)); } else { From d46a8394a3c4ae9afc4a08d4498613543ba2092b Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 16:00:18 +0300 Subject: [PATCH 047/180] Fixes compile error --- src/backend/distributed/deparser/citus_deparseutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 692d10e1ce6..3efca8575d1 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -35,7 +35,7 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo bool value = defGetBoolean(option); appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); } -#if PG_VERSION_NUM >= PG_VERSION_15 +#if PG_VERSION_NUM >= 150000 else if (strcmp(opt_formats[i].type, "object_id") == 0) { Oid value = defGetObjectId(option); From d778bb1ed6a6c8faabb8bb15686f46e848688d2a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 16:05:07 +0300 Subject: [PATCH 048/180] Adds PG_VERSION_15 constant --- src/backend/distributed/deparser/citus_deparseutils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 3efca8575d1..265988b4650 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -7,6 +7,7 @@ #include "utils/syscache.h" #include "utils/typcache.h" #include "distributed/deparser.h" +#include "distributed/pg_version_constants.h" void @@ -35,7 +36,7 @@ handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_fo bool value = defGetBoolean(option); appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); } -#if PG_VERSION_NUM >= 150000 +#if PG_VERSION_NUM >= PG_VERSION_15 else if (strcmp(opt_formats[i].type, "object_id") == 0) { Oid value = defGetObjectId(option); From 480c22acc2567fd1276bb465794c03154984627a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 18:10:28 +0300 Subject: [PATCH 049/180] Fixes the comment --- src/backend/distributed/commands/database.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index a12ff3f127e..62540689891 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -481,7 +481,9 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, UnmarkObjectDistributed(&dbAddress); } - /* we execute here to avoid EnsureCoordinator check in ExecuteDistributedDDLJob */ + /* ExecuteDistributedDDLJob could not be used since it depends on namespace and + * database does not have namespace. + */ bool outsideTransaction = false; List *taskList = CreateDDLTaskList(internalDropCommand->data, workerNodes, From d0ef13dc897d7d099548f17d206110f480d56d94 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 19:42:17 +0300 Subject: [PATCH 050/180] Fixes review comments --- src/backend/distributed/Makefile | 2 +- src/backend/distributed/commands/database.c | 22 ++----------------- src/backend/distributed/commands/index.c | 6 ++--- src/backend/distributed/commands/vacuum.c | 4 ++-- .../executor/executor_util_tasks.c | 4 ++-- .../distributed/utils/citus_outfuncs.c | 2 +- .../distributed/multi_physical_planner.h | 2 +- 7 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index f4f9ffd867d..8f28b04b025 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -18,7 +18,7 @@ generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$( DATA_built = $(generated_sql_files) # directories with source files -SUBDIRS = . commands connection database ddl deparser executor metadata operations planner progress relay safeclib shardsplit test transaction utils worker clock +SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib shardsplit test transaction utils worker clock # enterprise modules SUBDIRS += replication diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 62540689891..e676cee4de2 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -36,24 +36,6 @@ #include "distributed/listutils.h" #include "distributed/adaptive_executor.h" -/* macros to add DefElems to a list */ -#define DEFELEM_ADD_STRING(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *) makeString(value), -1); \ - options = lappend(options, elem); \ - } - -#define DEFELEM_ADD_BOOL(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *) makeBoolean(value), -1); \ - options = lappend(options, elem); \ - } - -#define DEFELEM_ADD_INT(options, key, value) \ - { \ - DefElem *elem = makeDefElem(key, (Node *) makeInteger(value), -1); \ - options = lappend(options, elem); \ - } static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); @@ -257,7 +239,7 @@ CreateDDLTaskList(char *command, List *workerNodeList, bool outsideTransaction) Task *task = CitusMakeNode(Task); task->taskType = DDL_TASK; SetTaskQueryStringList(task, commandList); - task->cannotBeExecutedInTransction = outsideTransaction; + task->cannotBeExecutedInTransaction = outsideTransaction; WorkerNode *workerNode = NULL; foreach_ptr(workerNode, workerNodeList) @@ -306,7 +288,7 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, /* * PostprocessCreatedbStmt creates the plan to synchronize CREATE DATABASE - * across nodes. We use the cannotBeExecutedInTransction option to avoid + * across nodes. We use the cannotBeExecutedInTransaction option to avoid * sending transaction blocks. */ List * diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 8271cc4f464..275f253b3b4 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -938,7 +938,7 @@ CreateIndexTaskList(IndexStmt *indexStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransction = indexStmt->concurrent; + task->cannotBeExecutedInTransaction = indexStmt->concurrent; taskList = lappend(taskList, task); @@ -983,7 +983,7 @@ CreateReindexTaskList(Oid relationId, ReindexStmt *reindexStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransction = + task->cannotBeExecutedInTransaction = IsReindexWithParam_compat(reindexStmt, "concurrently"); taskList = lappend(taskList, task); @@ -1309,7 +1309,7 @@ DropIndexTaskList(Oid relationId, Oid indexId, DropStmt *dropStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransction = dropStmt->concurrent; + task->cannotBeExecutedInTransaction = dropStmt->concurrent; taskList = lappend(taskList, task); diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index ee03aeae1a9..21638ba7f82 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -279,7 +279,7 @@ VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColum task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransction = ((vacuumParams.options) & VACOPT_VACUUM); + task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); taskList = lappend(taskList, task); } @@ -719,7 +719,7 @@ ExecuteUnqualifiedVacuumTasks(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumPa SetTaskQueryStringList(task, unqualifiedVacuumCommands); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; - task->cannotBeExecutedInTransction = ((vacuumParams.options) & VACOPT_VACUUM); + task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); bool hasPeerWorker = false; diff --git a/src/backend/distributed/executor/executor_util_tasks.c b/src/backend/distributed/executor/executor_util_tasks.c index abf72119600..483fd55a704 100644 --- a/src/backend/distributed/executor/executor_util_tasks.c +++ b/src/backend/distributed/executor/executor_util_tasks.c @@ -61,7 +61,7 @@ TaskListRequiresRollback(List *taskList) } Task *task = (Task *) linitial(taskList); - if (task->cannotBeExecutedInTransction) + if (task->cannotBeExecutedInTransaction) { /* vacuum, create index concurrently etc. */ return false; @@ -164,7 +164,7 @@ TaskListCannotBeExecutedInTransaction(List *taskList) Task *task = NULL; foreach_ptr(task, taskList) { - if (task->cannotBeExecutedInTransction) + if (task->cannotBeExecutedInTransaction) { return true; } diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index b4062751ace..9b4ac809c4e 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -535,7 +535,7 @@ OutTask(OUTFUNC_ARGS) WRITE_STRING_FIELD(fetchedExplainAnalyzePlan); WRITE_FLOAT_FIELD(fetchedExplainAnalyzeExecutionDuration, "%.2f"); WRITE_BOOL_FIELD(isLocalTableModification); - WRITE_BOOL_FIELD(cannotBeExecutedInTransction); + WRITE_BOOL_FIELD(cannotBeExecutedInTransaction); } diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index b7acc057412..35d83eb3388 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -329,7 +329,7 @@ typedef struct Task /* * Vacuum, create/drop/reindex concurrently cannot be executed in a transaction. */ - bool cannotBeExecutedInTransction; + bool cannotBeExecutedInTransaction; Const *partitionKeyValue; int colocationId; From d88a1e7785de0ce3cb1d2a9bd6202cacac089619 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 19:44:36 +0300 Subject: [PATCH 051/180] Fixes compile error --- src/backend/distributed/utils/citus_copyfuncs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index 7e1379ef32a..fe4429f04d0 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -326,7 +326,7 @@ CopyNodeTask(COPYFUNC_ARGS) COPY_STRING_FIELD(fetchedExplainAnalyzePlan); COPY_SCALAR_FIELD(fetchedExplainAnalyzeExecutionDuration); COPY_SCALAR_FIELD(isLocalTableModification); - COPY_SCALAR_FIELD(cannotBeExecutedInTransction); + COPY_SCALAR_FIELD(cannotBeExecutedInTransaction); } From fb6dd413ad6d390a8b2a857572a5d75745243cb4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 20:01:29 +0300 Subject: [PATCH 052/180] Fixes PostprocessCreateDatabaseStmt comment --- src/backend/distributed/commands/database.c | 22 ++++++++------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index e676cee4de2..17852bd9b53 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -287,9 +287,10 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, /* - * PostprocessCreatedbStmt creates the plan to synchronize CREATE DATABASE - * across nodes. We use the cannotBeExecutedInTransaction option to avoid - * sending transaction blocks. + * PostprocessCreatedbStmt is executed after the statement is applied to the local + * postgres instance. In this stage we can prepare the commands that need to be run on + * all workers to create the database. + * */ List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) @@ -381,10 +382,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) { createdb(NULL, (CreatedbStmt *) parseTree); } - else - { - /* TODO: check database properties */ - } + } else if (IsA(parseTree, DropdbStmt)) { @@ -393,14 +391,10 @@ citus_internal_database_command(PG_FUNCTION_ARGS) bool missingOk = false; Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - if (!OidIsValid(databaseOid)) - { - /* already dropped? */ - } - else + + if(OidIsValid(databaseOid)) { /* / * remove database from database shards * / */ - /* DeleteDatabaseShardByDatabaseIdLocally(databaseOid); */ DropDatabase(NULL, (DropdbStmt *) parseTree); } @@ -409,7 +403,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); } - + /* Below command rollbacks flags to the state before this session*/ AtEOXact_GUC(true, saveNestLevel); PG_RETURN_VOID(); From 7bdeb2f866248ace8fe7f2c76459c81b92af70bb Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 20:03:07 +0300 Subject: [PATCH 053/180] Fixes indentation --- src/backend/distributed/commands/database.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 17852bd9b53..762c19c79b5 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -382,7 +382,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) { createdb(NULL, (CreatedbStmt *) parseTree); } - } else if (IsA(parseTree, DropdbStmt)) { @@ -391,8 +390,8 @@ citus_internal_database_command(PG_FUNCTION_ARGS) bool missingOk = false; Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - if(OidIsValid(databaseOid)) + + if (OidIsValid(databaseOid)) { /* / * remove database from database shards * / */ @@ -403,6 +402,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); } + /* Below command rollbacks flags to the state before this session*/ AtEOXact_GUC(true, saveNestLevel); From f2092d2f440322b7cc065f59e89bb0a3b2c8a9bc Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 20:24:34 +0300 Subject: [PATCH 054/180] Clarifies the code --- src/backend/distributed/commands/database.c | 2 -- .../distributed/deparser/citus_deparseutils.c | 14 ++++++++++++-- .../distributed/deparser/deparse_database_stmts.c | 2 +- src/include/distributed/deparser.h | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 762c19c79b5..70fa871a0be 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -393,8 +393,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) if (OidIsValid(databaseOid)) { - /* / * remove database from database shards * / */ - DropDatabase(NULL, (DropdbStmt *) parseTree); } } diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 265988b4650..1dd8b33bc74 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -1,3 +1,4 @@ + #include "postgres.h" #include "utils/builtins.h" #include "commands/defrem.h" @@ -10,9 +11,18 @@ #include "distributed/pg_version_constants.h" +/** + * Convert a DefElem option to a SQL statement and append it to the given StringInfo buffer. + * + * @param buf The StringInfo buffer to append the SQL statement to. + * @param option The DefElem option to convert to a SQL statement. + * @param opt_formats The option format specification to use for the conversion. + * @param num_opt_formats The number of option formats in the opt_formats array. + */ void -handleOption(StringInfo buf, DefElem *option, const struct option_format *opt_formats, int - opt_formats_len) +optionToStatement(StringInfo buf, DefElem *option, const struct + option_format *opt_formats, int + opt_formats_len) { const char *name = option->defname; int i; diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index bc5d48f363e..a5a5584af2c 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -237,7 +237,7 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - handleOption(buf, option, create_database_option_formats, lengthof( + optionToStatement(buf, option, create_database_option_formats, lengthof( create_database_option_formats)); } } diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 428c91a5f06..375fd8a8d85 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -131,7 +131,7 @@ struct option_format }; -extern void handleOption(StringInfo buf, DefElem *option, const struct +extern void optionToStatement(StringInfo buf, DefElem *option, const struct option_format *opt_formats, int opt_formats_len); From 875f347238829f8cb791f338974ebd348cdb66be Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 9 Oct 2023 20:27:50 +0300 Subject: [PATCH 055/180] Fixes indentation --- src/backend/distributed/deparser/deparse_database_stmts.c | 2 +- src/include/distributed/deparser.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index a5a5584af2c..5365f072db2 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -238,7 +238,7 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { optionToStatement(buf, option, create_database_option_formats, lengthof( - create_database_option_formats)); + create_database_option_formats)); } } diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 375fd8a8d85..d47d3c18a64 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -132,8 +132,8 @@ struct option_format extern void optionToStatement(StringInfo buf, DefElem *option, const struct - option_format *opt_formats, int - opt_formats_len); + option_format *opt_formats, int + opt_formats_len); /* forward declarations for deparse_statistics_stmts.c */ From 8cc67921cb70dc68bf726d84f69a1a90d008f70b Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 00:12:38 +0300 Subject: [PATCH 056/180] Fixes review comments --- src/backend/distributed/commands/database.c | 27 +++++----- src/backend/distributed/metadata/distobject.c | 1 - .../distributed/metadata/metadata_cache.c | 49 +++++++++++++++++-- src/include/distributed/commands.h | 28 ++++++++--- .../create_drop_database_propagation.out | 16 ++++++ .../sql/create_drop_database_propagation.sql | 12 +++++ 6 files changed, 112 insertions(+), 21 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 70fa871a0be..06770111486 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -295,16 +295,13 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) { - if (EnableCreateDatabasePropagation) - { - EnsureCoordinator(); - } - if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } + EnsureCoordinator(); + CreatedbStmt *stmt = castNode(CreatedbStmt, node); char *databaseName = stmt->dbname; bool missingOk = false; @@ -371,6 +368,15 @@ citus_internal_database_command(PG_FUNCTION_ARGS) (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); + /* + * createdb() / DropDatabase() uses ParseState to report the error position for the + * input command and the position is reported to be 0 when it's provided as NULL. + * We're okay with that because we don't expect this UDF to be called with an incorrect + * DDL command. + * + */ + ParseState *pstate = NULL; + if (IsA(parseTree, CreatedbStmt)) { CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); @@ -380,7 +386,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) if (!OidIsValid(databaseOid)) { - createdb(NULL, (CreatedbStmt *) parseTree); + createdb(pstate, (CreatedbStmt *) parseTree); } } else if (IsA(parseTree, DropdbStmt)) @@ -393,7 +399,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) if (OidIsValid(databaseOid)) { - DropDatabase(NULL, (DropdbStmt *) parseTree); + DropDatabase(pstate, (DropdbStmt *) parseTree); } } else @@ -417,6 +423,8 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } + EnsureCoordinator(); + DropdbStmt *stmt = (DropdbStmt *) node; char *databaseName = stmt->dbname; bool missingOk = true; @@ -450,10 +458,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, /* Delete from pg_dist_object */ - if (IsObjectDistributed(&dbAddress)) - { - UnmarkObjectDistributed(&dbAddress); - } + UnmarkObjectDistributed(&dbAddress); /* ExecuteDistributedDDLJob could not be used since it depends on namespace and * database does not have namespace. diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index a025ba73fa3..c420e6ec390 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -53,7 +53,6 @@ static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, Datum *paramValues); -bool IsObjectDistributed(const ObjectAddress *address); PG_FUNCTION_INFO_V1(citus_unmark_object_distributed); PG_FUNCTION_INFO_V1(master_unmark_object_distributed); diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 55ec63a6abb..55d0f11c57f 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -310,6 +310,7 @@ static HeapTuple LookupDistPartitionTuple(Relation pgDistPartition, Oid relation static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, Oid *columnTypeId, int32 *columnTypeMod, Oid *intervalTypeId, int32 *intervalTypeMod); +static void CachedNamespaceLookup(const char *nspname, Oid *cachedOid); static void CachedRelationLookup(const char *relationName, Oid *cachedOid); static void CachedRelationLookupExtended(const char *relationName, Oid *cachedOid, bool missing_ok); @@ -2769,6 +2770,15 @@ DistRebalanceStrategyRelationId(void) } +/* return the oid of citus namespace */ +Oid +CitusCatalogNamespaceId(void) +{ + CachedNamespaceLookup("citus", &MetadataCache.citusCatalogNamespaceId); + return MetadataCache.citusCatalogNamespaceId; +} + + /* return oid of pg_dist_object relation */ Oid DistObjectRelationId(void) @@ -2795,14 +2805,12 @@ DistObjectRelationId(void) true); if (!OidIsValid(MetadataCache.distObjectRelationId)) { - Oid citusNamespaceId = get_namespace_oid("citus", false); - /* * We can only ever reach here while we are creating/altering our extension before * the table is moved to pg_catalog. */ CachedRelationNamespaceLookupExtended("pg_dist_object", - citusNamespaceId, + CitusCatalogNamespaceId(), &MetadataCache.distObjectRelationId, false); } @@ -2836,6 +2844,17 @@ DistObjectPrimaryKeyIndexId(void) &MetadataCache.distObjectPrimaryKeyIndexId, true); + if (!OidIsValid(MetadataCache.distObjectPrimaryKeyIndexId)) + { + /* + * We can only ever reach here while we are creating/altering our extension before + * the table is moved to pg_catalog. + */ + CachedRelationNamespaceLookupExtended("pg_dist_object_pkey", + CitusCatalogNamespaceId(), + &MetadataCache.distObjectPrimaryKeyIndexId, + false); + } return MetadataCache.distObjectPrimaryKeyIndexId; } @@ -5401,6 +5420,30 @@ DeformedDistShardTupleToShardInterval(Datum *datumArray, bool *isNullArray, } +/* + * CachedNamespaceLookup performs a cached lookup for the namespace (schema), with the + * result cached in cachedOid. + */ +static void +CachedNamespaceLookup(const char *nspname, Oid *cachedOid) +{ + /* force callbacks to be registered, so we always get notified upon changes */ + InitializeCaches(); + + if (*cachedOid == InvalidOid) + { + *cachedOid = get_namespace_oid(nspname, true); + + if (*cachedOid == InvalidOid) + { + ereport(ERROR, (errmsg( + "cache lookup failed for namespace %s, called too early?", + nspname))); + } + } +} + + /* * CachedRelationLookup performs a cached lookup for the relation * relationName, with the result cached in *cachedOid. diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 36e41237896..22c35a694ab 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -22,6 +22,7 @@ #include "tcop/utility.h" #include "utils/acl.h" + extern bool AddAllLocalTablesToMetadata; extern bool EnableSchemaBasedSharding; @@ -57,6 +58,7 @@ typedef enum DistOpsOperationType DIST_OPS_DROP, } DistOpsOperationType; + /* * DistributeObjectOps specifies handlers for node/object type pairs. * Instances of this type should all be declared in deparse.c. @@ -77,11 +79,11 @@ typedef enum DistOpsOperationType */ typedef struct DistributeObjectOps { - char *(*deparse)(Node *); + char * (*deparse)(Node *); void (*qualify)(Node *); - List *(*preprocess)(Node *, const char *, ProcessUtilityContext); - List *(*postprocess)(Node *, const char *); - List *(*address)(Node *, bool, bool); + List * (*preprocess)(Node *, const char *, ProcessUtilityContext); + List * (*postprocess)(Node *, const char *); + List * (*address)(Node *, bool, bool); bool markDistributed; /* fields used by common implementations, omitted for specialized implementations */ @@ -138,6 +140,7 @@ typedef enum ExtractForeignKeyConstraintsMode INCLUDE_SINGLE_SHARD_TABLES } ExtractForeignKeyConstraintMode; + /* * Flags that can be passed to GetForeignKeyIdsForColumn to * indicate whether relationId argument should match: @@ -156,6 +159,7 @@ typedef enum SearchForeignKeyColumnFlags /* callers can also pass union of above flags */ } SearchForeignKeyColumnFlags; + typedef enum TenantOperation { TENANT_UNDISTRIBUTE_TABLE = 0, @@ -193,9 +197,11 @@ extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); + /* call.c */ extern bool CallDistributedProcedureRemotely(CallStmt *callStmt, DestReceiver *dest); + /* collation.c - forward declarations */ extern char * CreateCollationDDL(Oid collationId); extern List * CreateCollationDDLsIdempotent(Oid collationId); @@ -224,6 +230,7 @@ extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *que ProcessUtilityContext processUtilityContext); + extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); @@ -246,6 +253,7 @@ extern List * RenameDomainStmtObjectAddress(Node *node, bool missing_ok, bool extern CreateDomainStmt * RecreateDomainStmt(Oid domainOid); extern Oid get_constraint_typid(Oid conoid); + /* extension.c - forward declarations */ extern bool IsDropCitusExtensionStmt(Node *parsetree); extern List * GetDependentFDWsToExtension(Oid extensionId); @@ -324,11 +332,13 @@ extern Oid GetReferencedTableId(Oid foreignKeyId); extern Oid GetReferencingTableId(Oid foreignKeyId); extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId); + /* foreign_data_wrapper.c - forward declarations */ extern List * PreprocessGrantOnFDWStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern Acl * GetPrivilegesForFDW(Oid FDWOid); + /* foreign_server.c - forward declarations */ extern List * PreprocessGrantOnForeignServerStmt(Node *node, const char *queryString, ProcessUtilityContext @@ -339,15 +349,17 @@ extern List * AlterForeignServerStmtObjectAddress(Node *node, bool missing_ok, b isPostprocess); extern List * RenameForeignServerStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List * AlterForeignServerOwnerStmtObjectAddress(Node *node, bool + missing_ok, bool isPostprocess); extern List * GetForeignServerCreateDDLCommand(Oid serverId); + /* foreign_table.c - forward declarations */ extern List * PreprocessAlterForeignTableSchemaStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); + /* function.c - forward declarations */ extern List * PreprocessCreateFunctionStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); @@ -377,12 +389,14 @@ extern List * PreprocessGrantOnFunctionStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern List * PostprocessGrantOnFunctionStmt(Node *node, const char *queryString); + /* grant.c - forward declarations */ extern List * PreprocessGrantStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); extern void deparsePrivileges(StringInfo privsString, GrantStmt *grantStmt); extern void deparseGrantees(StringInfo granteesString, GrantStmt *grantStmt); + /* index.c - forward declarations */ extern bool IsIndexRenameStmt(RenameStmt *renameStmt); extern List * PreprocessIndexStmt(Node *createIndexStatement, @@ -463,6 +477,7 @@ extern void ErrorIfUnsupportedRenameStmt(RenameStmt *renameStmt); extern List * PreprocessRenameAttributeStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); + /* role.c - forward declarations*/ extern List * PostprocessAlterRoleStmt(Node *stmt, const char *queryString); extern List * PreprocessAlterRoleSetStmt(Node *stmt, const char *queryString, @@ -585,6 +600,7 @@ extern List * GetAlterIndexStatisticsCommands(Oid indexOid); /* subscription.c - forward declarations */ extern Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); + /* table.c - forward declarations */ extern List * PreprocessDropTableStmt(Node *stmt, const char *queryString, ProcessUtilityContext processUtilityContext); diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index e578cd2ebdf..be85e50c0e7 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -102,6 +102,22 @@ WHERE datname = 'mydatabase'; (0 rows) \c - - - :master_port +--tests for special characters in database name +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +create database "mydatabase#1'2"; +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE "mydatabase#1''2"') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE "mydatabase#1''2"') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database "mydatabase#1'2"; +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE "mydatabase#1''2"') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE "mydatabase#1''2"') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +--clean up resources created by this test drop tablespace create_drop_db_tablespace; \c - - - :worker_1_port drop tablespace create_drop_db_tablespace; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 415a03f9743..5e2166f7740 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -94,6 +94,18 @@ WHERE datname = 'mydatabase'; \c - - - :master_port +--tests for special characters in database name +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + +create database "mydatabase#1'2"; + +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database "mydatabase#1'2"; + +--clean up resources created by this test + drop tablespace create_drop_db_tablespace; \c - - - :worker_1_port From a274ffcb51a717cbfff914ab7f4b0fafe5f3ce5c Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 01:36:01 +0300 Subject: [PATCH 057/180] Adds If exists statement for drop database --- src/backend/distributed/deparser/deparse_database_stmts.c | 4 +++- .../regress/expected/create_drop_database_propagation.out | 6 +++--- src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 5365f072db2..7c7544694c9 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -259,8 +259,10 @@ DeparseCreateDatabaseStmt(Node *node) static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { + char *if_exists_statement = stmt->missing_ok ? "IF EXISTS" : ""; appendStringInfo(buf, - "DROP DATABASE %s", + "DROP DATABASE %s %s", + if_exists_statement, quote_identifier(stmt->dbname)); DefElem *option = NULL; diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index be85e50c0e7..37829a6eeb6 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -112,10 +112,10 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE "mydatabase#1''2"') DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database "mydatabase#1'2"; -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE "mydatabase#1''2"') +drop database if exists "mydatabase#1'2"; +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE IF EXISTS "mydatabase#1''2"') DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE "mydatabase#1''2"') +NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE IF EXISTS "mydatabase#1''2"') DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx --clean up resources created by this test drop tablespace create_drop_db_tablespace; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 5e2166f7740..d846540541f 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -102,7 +102,7 @@ set citus.grep_remote_commands = '%CREATE DATABASE%'; create database "mydatabase#1'2"; set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database "mydatabase#1'2"; +drop database if exists "mydatabase#1'2"; --clean up resources created by this test From c31479d2eb6ffe3af20dc5be6be4f60e01472af9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 03:10:17 +0300 Subject: [PATCH 058/180] Fixes review notes --- src/backend/distributed/commands/database.c | 127 +++--------------- src/backend/distributed/commands/index.c | 6 +- src/backend/distributed/commands/vacuum.c | 4 +- .../distributed/deparser/citus_deparseutils.c | 12 +- .../deparser/deparse_database_stmts.c | 32 ++--- .../executor/executor_util_tasks.c | 4 +- .../distributed/utils/citus_copyfuncs.c | 2 +- .../distributed/utils/citus_outfuncs.c | 2 +- src/include/distributed/deparser.h | 11 +- .../distributed/multi_physical_planner.h | 2 +- 10 files changed, 57 insertions(+), 145 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 06770111486..93cf87b4282 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -39,8 +39,6 @@ static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); -static List * CreateDDLTaskList(char *command, List *workerNodeList, - bool outsideTransaction); PG_FUNCTION_INFO_V1(citus_internal_database_command); static Oid get_database_owner(Oid db_oid); @@ -226,36 +224,6 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString, #endif -/* - * CreateDDLTaskList creates a task list for running a single DDL command. - */ -static List * -CreateDDLTaskList(char *command, List *workerNodeList, bool outsideTransaction) -{ - List *commandList = list_make3(DISABLE_DDL_PROPAGATION, - command, - ENABLE_DDL_PROPAGATION); - - Task *task = CitusMakeNode(Task); - task->taskType = DDL_TASK; - SetTaskQueryStringList(task, commandList); - task->cannotBeExecutedInTransaction = outsideTransaction; - - WorkerNode *workerNode = NULL; - foreach_ptr(workerNode, workerNodeList) - { - ShardPlacement *targetPlacement = CitusMakeNode(ShardPlacement); - targetPlacement->nodeName = workerNode->workerName; - targetPlacement->nodePort = workerNode->workerPort; - targetPlacement->groupId = workerNode->groupId; - task->taskPlacementList = lappend(task->taskPlacementList, - targetPlacement); - } - - return list_make1(task); -} - - /* * PreprocessAlterDatabaseSetStmt is executed before the statement is applied to the local * postgres instance. @@ -295,55 +263,25 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) { - if (!EnableCreateDatabasePropagation || !ShouldPropagate()) + if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); - CreatedbStmt *stmt = castNode(CreatedbStmt, node); - char *databaseName = stmt->dbname; - bool missingOk = false; - Oid databaseOid = get_database_oid(databaseName, missingOk); + char *createDatabaseCommand = DeparseTreeNode(node); - /* - * TODO: try to reuse regular DDL infrastructure - * - * We do not do this right now because of the AssignDatabaseToShard at the end. - */ - List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, - RowShareLock); - if (list_length(workerNodes) > 0) - { - char *createDatabaseCommand = DeparseTreeNode(node); - - StringInfo internalCreateCommand = makeStringInfo(); - appendStringInfo(internalCreateCommand, - "SELECT pg_catalog.citus_internal_database_command(%s)", - quote_literal_cstr(createDatabaseCommand)); - - /* - * For the moment, we run CREATE DATABASE in 2PC, though that prevents - * us from immediately doing a pg_dump | pg_restore when dealing with - * a remote template database. - */ - bool outsideTransaction = false; - - List *taskList = CreateDDLTaskList(internalCreateCommand->data, workerNodes, - outsideTransaction); - - bool localExecutionSupported = false; - ExecuteUtilityTaskList(taskList, localExecutionSupported); - } - - /* synchronize pg_dist_object records */ - ObjectAddress dbAddress = { 0 }; - ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); - MarkObjectDistributed(&dbAddress); + StringInfo internalCreateCommand = makeStringInfo(); + appendStringInfo(internalCreateCommand, + "SELECT pg_catalog.citus_internal_database_command(%s)", + quote_literal_cstr(createDatabaseCommand)); + List *commands = list_make3(DISABLE_DDL_PROPAGATION, + (void *) internalCreateCommand->data, + ENABLE_DDL_PROPAGATION); - return NIL; + return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } @@ -418,37 +356,13 @@ List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { - if (!EnableCreateDatabasePropagation || !ShouldPropagate()) + if (!ShouldPropagate()) { return NIL; } EnsureCoordinator(); - DropdbStmt *stmt = (DropdbStmt *) node; - char *databaseName = stmt->dbname; - bool missingOk = true; - Oid databaseOid = get_database_oid(databaseName, missingOk); - if (databaseOid == InvalidOid) - { - /* let regular ProcessUtility deal with IF NOT EXISTS */ - return NIL; - } - - ObjectAddress dbAddress = { 0 }; - ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); - if (!IsObjectDistributed(&dbAddress)) - { - return NIL; - } - - List *workerNodes = TargetWorkerSetNodeList(NON_COORDINATOR_METADATA_NODES, - RowShareLock); - if (list_length(workerNodes) == 0) - { - return NIL; - } - char *dropDatabaseCommand = DeparseTreeNode(node); StringInfo internalDropCommand = makeStringInfo(); @@ -456,20 +370,9 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, "SELECT pg_catalog.citus_internal_database_command(%s)", quote_literal_cstr(dropDatabaseCommand)); - /* Delete from pg_dist_object */ - - UnmarkObjectDistributed(&dbAddress); - - /* ExecuteDistributedDDLJob could not be used since it depends on namespace and - * database does not have namespace. - */ - - bool outsideTransaction = false; - List *taskList = CreateDDLTaskList(internalDropCommand->data, workerNodes, - outsideTransaction); - - bool localExecutionSupported = false; - ExecuteUtilityTaskList(taskList, localExecutionSupported); + List *commands = list_make3(DISABLE_DDL_PROPAGATION, + (void *) internalDropCommand->data, + ENABLE_DDL_PROPAGATION); - return NIL; + return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 275f253b3b4..8271cc4f464 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -938,7 +938,7 @@ CreateIndexTaskList(IndexStmt *indexStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransaction = indexStmt->concurrent; + task->cannotBeExecutedInTransction = indexStmt->concurrent; taskList = lappend(taskList, task); @@ -983,7 +983,7 @@ CreateReindexTaskList(Oid relationId, ReindexStmt *reindexStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransaction = + task->cannotBeExecutedInTransction = IsReindexWithParam_compat(reindexStmt, "concurrently"); taskList = lappend(taskList, task); @@ -1309,7 +1309,7 @@ DropIndexTaskList(Oid relationId, Oid indexId, DropStmt *dropStmt) task->dependentTaskList = NULL; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransaction = dropStmt->concurrent; + task->cannotBeExecutedInTransction = dropStmt->concurrent; taskList = lappend(taskList, task); diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index 21638ba7f82..ee03aeae1a9 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -279,7 +279,7 @@ VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColum task->replicationModel = REPLICATION_MODEL_INVALID; task->anchorShardId = shardId; task->taskPlacementList = ActiveShardPlacementList(shardId); - task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); + task->cannotBeExecutedInTransction = ((vacuumParams.options) & VACOPT_VACUUM); taskList = lappend(taskList, task); } @@ -719,7 +719,7 @@ ExecuteUnqualifiedVacuumTasks(VacuumStmt *vacuumStmt, CitusVacuumParams vacuumPa SetTaskQueryStringList(task, unqualifiedVacuumCommands); task->dependentTaskList = NULL; task->replicationModel = REPLICATION_MODEL_INVALID; - task->cannotBeExecutedInTransaction = ((vacuumParams.options) & VACOPT_VACUUM); + task->cannotBeExecutedInTransction = ((vacuumParams.options) & VACOPT_VACUUM); bool hasPeerWorker = false; diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 1dd8b33bc74..52d96930e6b 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -31,36 +31,36 @@ optionToStatement(StringInfo buf, DefElem *option, const struct { if (strcmp(name, opt_formats[i].name) == 0) { - if (strcmp(opt_formats[i].type, "string") == 0) + if (opt_formats[i].type == OPTION_FORMAT_STRING) { char *value = defGetString(option); appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); } - else if (strcmp(opt_formats[i].type, "integer") == 0) + else if (opt_formats[i].type == OPTION_FORMAT_INTEGER) { int32 value = defGetInt32(option); appendStringInfo(buf, opt_formats[i].format, value); } - else if (strcmp(opt_formats[i].type, "boolean") == 0) + else if (opt_formats[i].type == OPTION_FORMAT_BOOLEAN) { bool value = defGetBoolean(option); appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); } #if PG_VERSION_NUM >= PG_VERSION_15 - else if (strcmp(opt_formats[i].type, "object_id") == 0) + else if (opt_formats[i].type == OPTION_FORMAT_OBJECT_ID) { Oid value = defGetObjectId(option); appendStringInfo(buf, opt_formats[i].format, value); } #endif - else if (strcmp(opt_formats[i].type, "literal_cstr") == 0) + else if (opt_formats[i].type == OPTION_FORMAT_LITERAL_CSTR) { char *value = defGetString(option); appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value)); } else { - elog(ERROR, "unrecognized option type: %s", opt_formats[i].type); + elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); } break; } diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 7c7544694c9..bf98b962251 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -31,22 +31,22 @@ static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendDefElemConnLimit(StringInfo buf, DefElem *def); const struct option_format create_database_option_formats[] = { - { "template", " TEMPLATE %s", "string" }, - { "owner", " OWNER %s", "string" }, - { "tablespace", " TABLESPACE %s", "string" }, - { "connection_limit", " CONNECTION LIMIT %d", "integer" }, - { "encoding", " ENCODING %s", "literal_cstr" }, - { "locale", " LOCALE %s", "literal_cstr" }, - { "lc_collate", " LC_COLLATE %s", "literal_cstr" }, - { "lc_ctype", " LC_CTYPE %s", "literal_cstr" }, - { "icu_locale", " ICU_LOCALE %s", "literal_cstr" }, - { "icu_rules", " ICU_RULES %s", "literal_cstr" }, - { "locale_provider", " LOCALE_PROVIDER %s", "literal_cstr" }, - { "is_template", " IS_TEMPLATE %s", "boolean" }, - { "allow_connections", " ALLOW_CONNECTIONS %s", "boolean" }, - { "collation_version", " COLLATION_VERSION %s", "literal_cstr" }, - { "strategy", " STRATEGY %s", "literal_cstr" }, - { "oid", " OID %d", "object_id" }, + { "owner", " OWNER %s", OPTION_FORMAT_STRING }, + { "template", " TEMPLATE %s", OPTION_FORMAT_STRING }, + { "encoding", " ENCODING %s", OPTION_FORMAT_LITERAL_CSTR }, + { "strategy", " STRATEGY %s", OPTION_FORMAT_LITERAL_CSTR }, + { "locale", " LOCALE %s", OPTION_FORMAT_LITERAL_CSTR }, + { "lc_collate", " LC_COLLATE %s", OPTION_FORMAT_LITERAL_CSTR }, + { "lc_ctype", " LC_CTYPE %s", OPTION_FORMAT_LITERAL_CSTR }, + { "icu_locale", " ICU_LOCALE %s", OPTION_FORMAT_LITERAL_CSTR }, + { "icu_rules", " ICU_RULES %s", OPTION_FORMAT_LITERAL_CSTR }, + { "locale_provider", " LOCALE_PROVIDER %s", OPTION_FORMAT_LITERAL_CSTR }, + { "collation_version", " COLLATION_VERSION %s", OPTION_FORMAT_LITERAL_CSTR }, + { "tablespace", " TABLESPACE %s", OPTION_FORMAT_STRING }, + { "allow_connections", " ALLOW_CONNECTIONS %s", OPTION_FORMAT_BOOLEAN }, + { "connection_limit", " CONNECTION LIMIT %d", OPTION_FORMAT_INTEGER }, + { "is_template", " IS_TEMPLATE %s", OPTION_FORMAT_BOOLEAN }, + { "oid", " OID %d", OPTION_FORMAT_OBJECT_ID } }; char * diff --git a/src/backend/distributed/executor/executor_util_tasks.c b/src/backend/distributed/executor/executor_util_tasks.c index 483fd55a704..abf72119600 100644 --- a/src/backend/distributed/executor/executor_util_tasks.c +++ b/src/backend/distributed/executor/executor_util_tasks.c @@ -61,7 +61,7 @@ TaskListRequiresRollback(List *taskList) } Task *task = (Task *) linitial(taskList); - if (task->cannotBeExecutedInTransaction) + if (task->cannotBeExecutedInTransction) { /* vacuum, create index concurrently etc. */ return false; @@ -164,7 +164,7 @@ TaskListCannotBeExecutedInTransaction(List *taskList) Task *task = NULL; foreach_ptr(task, taskList) { - if (task->cannotBeExecutedInTransaction) + if (task->cannotBeExecutedInTransction) { return true; } diff --git a/src/backend/distributed/utils/citus_copyfuncs.c b/src/backend/distributed/utils/citus_copyfuncs.c index fe4429f04d0..7e1379ef32a 100644 --- a/src/backend/distributed/utils/citus_copyfuncs.c +++ b/src/backend/distributed/utils/citus_copyfuncs.c @@ -326,7 +326,7 @@ CopyNodeTask(COPYFUNC_ARGS) COPY_STRING_FIELD(fetchedExplainAnalyzePlan); COPY_SCALAR_FIELD(fetchedExplainAnalyzeExecutionDuration); COPY_SCALAR_FIELD(isLocalTableModification); - COPY_SCALAR_FIELD(cannotBeExecutedInTransaction); + COPY_SCALAR_FIELD(cannotBeExecutedInTransction); } diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index 9b4ac809c4e..b4062751ace 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -535,7 +535,7 @@ OutTask(OUTFUNC_ARGS) WRITE_STRING_FIELD(fetchedExplainAnalyzePlan); WRITE_FLOAT_FIELD(fetchedExplainAnalyzeExecutionDuration, "%.2f"); WRITE_BOOL_FIELD(isLocalTableModification); - WRITE_BOOL_FIELD(cannotBeExecutedInTransaction); + WRITE_BOOL_FIELD(cannotBeExecutedInTransction); } diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index d47d3c18a64..66ead2b4d0b 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -127,9 +127,18 @@ struct option_format { const char *name; const char *format; - const char *type; + const int type; }; +typedef enum OptionFormatType +{ + OPTION_FORMAT_STRING, + OPTION_FORMAT_LITERAL_CSTR, + OPTION_FORMAT_BOOLEAN, + OPTION_FORMAT_INTEGER, + OPTION_FORMAT_OBJECT_ID +} OptionFormatType; + extern void optionToStatement(StringInfo buf, DefElem *option, const struct option_format *opt_formats, int diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 35d83eb3388..b7acc057412 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -329,7 +329,7 @@ typedef struct Task /* * Vacuum, create/drop/reindex concurrently cannot be executed in a transaction. */ - bool cannotBeExecutedInTransaction; + bool cannotBeExecutedInTransction; Const *partitionKeyValue; int colocationId; From 333f77a26760edccf24bac3a6bdd931a64a2fec0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 04:16:20 +0300 Subject: [PATCH 059/180] Fixes distributed_object management --- src/backend/distributed/commands/database.c | 32 +++++++++++++++++++ .../commands/distribute_object_ops.c | 4 +-- src/include/distributed/commands.h | 3 ++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 93cf87b4282..bc3e197a40d 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -363,6 +363,25 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); + DropdbStmt *stmt = (DropdbStmt *) node; + + Oid databaseOid = get_database_oid(stmt->dbname, stmt->missing_ok); + + if (databaseOid == InvalidOid) + { + /* let regular ProcessUtility deal with IF NOT EXISTS */ + return NIL; + } + + ObjectAddress dbAddress = { 0 }; + ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); + if (!IsObjectDistributed(&dbAddress)) + { + return NIL; + } + + UnmarkObjectDistributed(&dbAddress); + char *dropDatabaseCommand = DeparseTreeNode(node); StringInfo internalDropCommand = makeStringInfo(); @@ -370,9 +389,22 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, "SELECT pg_catalog.citus_internal_database_command(%s)", quote_literal_cstr(dropDatabaseCommand)); + List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) internalDropCommand->data, ENABLE_DDL_PROPAGATION); return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + + +List * +CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) +{ + CreatedbStmt *stmt = castNode(CreatedbStmt, node); + Oid databaseOid = get_database_oid(stmt->dbname, missing_ok); + ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbAddress, DatabaseRelationId, databaseOid); + + return list_make1(dbAddress); +} diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index ef7d486b5c0..49a96e01653 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -473,8 +473,8 @@ static DistributeObjectOps Database_Create = { .postprocess = PostprocessCreateDatabaseStmt, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_CREATE, - .address = NULL, - .markDistributed = false, + .address = CreateDatabaseStmtObjectAddress, + .markDistributed = true, }; static DistributeObjectOps Database_Drop = { diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 22c35a694ab..b1f65177e35 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -230,6 +230,9 @@ extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *que ProcessUtilityContext processUtilityContext); +extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); + extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); From abf9dd9e596f7d2aacd365db9af19a7454d8a706 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 04:36:49 +0300 Subject: [PATCH 060/180] Adds EnableCreateDatabasePropagation check --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index bc3e197a40d..d177cdcea81 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -263,7 +263,7 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) { - if (!ShouldPropagate()) + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } From ca5e2348215b2ef25ca6ce3d79b71836f0c14956 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 13 Oct 2023 04:46:53 +0300 Subject: [PATCH 061/180] Adds EnableCreateDatabasePropagation for drop db --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index d177cdcea81..8aee9213ff3 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -356,7 +356,7 @@ List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { - if (!ShouldPropagate()) + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } From 3a6fdada111c1eb70d1f1d594d70af6a667cf601 Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 14 Oct 2023 05:35:34 +0300 Subject: [PATCH 062/180] Changes if to switch statements --- .../distributed/deparser/citus_deparseutils.c | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 52d96930e6b..f96244a2704 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -31,36 +31,38 @@ optionToStatement(StringInfo buf, DefElem *option, const struct { if (strcmp(name, opt_formats[i].name) == 0) { - if (opt_formats[i].type == OPTION_FORMAT_STRING) - { - char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); - } - else if (opt_formats[i].type == OPTION_FORMAT_INTEGER) - { - int32 value = defGetInt32(option); - appendStringInfo(buf, opt_formats[i].format, value); - } - else if (opt_formats[i].type == OPTION_FORMAT_BOOLEAN) - { - bool value = defGetBoolean(option); - appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); - } -#if PG_VERSION_NUM >= PG_VERSION_15 - else if (opt_formats[i].type == OPTION_FORMAT_OBJECT_ID) - { - Oid value = defGetObjectId(option); - appendStringInfo(buf, opt_formats[i].format, value); - } -#endif - else if (opt_formats[i].type == OPTION_FORMAT_LITERAL_CSTR) - { - char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value)); - } - else - { - elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); + switch (opt_formats[i].type) { + case OPTION_FORMAT_STRING: { + char *value = defGetString(option); + appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); + break; + } + case OPTION_FORMAT_INTEGER: { + int32 value = defGetInt32(option); + appendStringInfo(buf, opt_formats[i].format, value); + break; + } + case OPTION_FORMAT_BOOLEAN: { + bool value = defGetBoolean(option); + appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); + break; + } + #if PG_VERSION_NUM >= PG_VERSION_15 + case OPTION_FORMAT_OBJECT_ID: { + Oid value = defGetObjectId(option); + appendStringInfo(buf, opt_formats[i].format, value); + break; + } + #endif + case OPTION_FORMAT_LITERAL_CSTR: { + char *value = defGetString(option); + appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value)); + break; + } + default: { + elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); + break; + } } break; } From 674fd3226c293e94c3fe56ac0739a2b7011a843b Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 14 Oct 2023 19:50:32 +0300 Subject: [PATCH 063/180] Sets enable_create_database_propagation true --- src/backend/distributed/shared_library_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 00cdb0027e2..886517464e5 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1268,7 +1268,7 @@ RegisterCitusConfigVariables(void) "and DROP DATABASE statements to workers"), NULL, &EnableCreateDatabasePropagation, - false, + true, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); From c6d1ef9e41a25255bc8ca082680cd58f6866d5f0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 14 Oct 2023 20:19:18 +0300 Subject: [PATCH 064/180] Rollbacks enable_create_database_propagation --- .../distributed/deparser/citus_deparseutils.c | 33 +++++++++++++------ src/backend/distributed/shared_library_init.c | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index f96244a2704..6492c14f2ce 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -31,40 +31,53 @@ optionToStatement(StringInfo buf, DefElem *option, const struct { if (strcmp(name, opt_formats[i].name) == 0) { - switch (opt_formats[i].type) { - case OPTION_FORMAT_STRING: { + switch (opt_formats[i].type) + { + case OPTION_FORMAT_STRING: + { char *value = defGetString(option); appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); break; } - case OPTION_FORMAT_INTEGER: { + + case OPTION_FORMAT_INTEGER: + { int32 value = defGetInt32(option); appendStringInfo(buf, opt_formats[i].format, value); break; } - case OPTION_FORMAT_BOOLEAN: { + + case OPTION_FORMAT_BOOLEAN: + { bool value = defGetBoolean(option); - appendStringInfo(buf, opt_formats[i].format, value ? "true" : "false"); + appendStringInfo(buf, opt_formats[i].format, value ? "true" : + "false"); break; } + #if PG_VERSION_NUM >= PG_VERSION_15 - case OPTION_FORMAT_OBJECT_ID: { + case OPTION_FORMAT_OBJECT_ID: + { Oid value = defGetObjectId(option); appendStringInfo(buf, opt_formats[i].format, value); break; } + #endif - case OPTION_FORMAT_LITERAL_CSTR: { + case OPTION_FORMAT_LITERAL_CSTR: + { char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr(value)); + appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr( + value)); break; } - default: { + + default: + { elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); break; } } - break; } } } diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 886517464e5..00cdb0027e2 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1268,7 +1268,7 @@ RegisterCitusConfigVariables(void) "and DROP DATABASE statements to workers"), NULL, &EnableCreateDatabasePropagation, - true, + false, PGC_USERSET, GUC_STANDARD, NULL, NULL, NULL); From a497a782399eae23178dd038a5a7686823c359a1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 14 Oct 2023 20:39:39 +0300 Subject: [PATCH 065/180] Fixes review comments --- .../regress/expected/create_drop_database_propagation.out | 6 +++--- src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 37829a6eeb6..32552d01ab7 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -21,15 +21,15 @@ CREATE DATABASE mydatabase IS_TEMPLATE = false; SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pd.datcollate , pd. datctype , pd.datacl, pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid join pg_tablespace pt on pd.dattablespace = pt.oid WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | tablespace --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_tablespace (1 row) \c - - - :worker_1_port diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index d846540541f..d2195659570 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -29,7 +29,7 @@ CREATE DATABASE mydatabase SELECT pd.datname, pd.encoding, pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, +pd.datcollate , pd. datctype , pd.datacl, pa.rolname AS database_owner, pt.spcname AS tablespace FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid From 1e91b144e2ca5a16cbc1d66f6b10a1b1ced05ac4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Sat, 14 Oct 2023 20:49:26 +0300 Subject: [PATCH 066/180] Fixed review issues --- .../distributed/sql/citus--12.1-1--12.2-1.sql | 14 +------------- .../citus_internal_database_command/12.2-1.sql | 10 ++++++++++ .../citus_internal_database_command/latest.sql | 10 ++++++++++ 3 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql create mode 100644 src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql diff --git a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql index 5d869f40e0d..578a182eff4 100644 --- a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql @@ -1,16 +1,4 @@ -- citus--12.1-1--12.2-1 - --- --- citus_internal_database_command creates a database according to the given command. - -CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) - RETURNS void - LANGUAGE C - STRICT -AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; -COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS - 'run a database command without transaction block restrictions'; - -- bump version to 12.2-1 - +#include "udfs/citus_internal_database_command/12.2-1.sql" #include "udfs/citus_add_rebalance_strategy/12.2-1.sql" diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql new file mode 100644 index 00000000000..232e3ad143e --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql @@ -0,0 +1,10 @@ +-- +-- citus_internal_database_command creates a database according to the given command. + +CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) + RETURNS void + LANGUAGE C + STRICT +AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; +COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS + 'run a database command without transaction block restrictions'; diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql new file mode 100644 index 00000000000..232e3ad143e --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql @@ -0,0 +1,10 @@ +-- +-- citus_internal_database_command creates a database according to the given command. + +CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) + RETURNS void + LANGUAGE C + STRICT +AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; +COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS + 'run a database command without transaction block restrictions'; From dad81042a5996d54199e5b73a550857049cb2f3d Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 23 Oct 2023 10:01:18 +0300 Subject: [PATCH 067/180] Adds undistribute sql --- src/backend/distributed/commands/database.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 8aee9213ff3..7080eeba82a 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -240,6 +240,8 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, return NIL; } + + AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); EnsureCoordinator(); @@ -351,6 +353,13 @@ citus_internal_database_command(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +static char * GetUnmarkDatabaseDistributedSql(char* dbName){ + StringInfoData pg_dist_object_delete = { 0 }; + initStringInfo(&pg_dist_object_delete); + appendStringInfo(&pg_dist_object_delete, "delete from pg_dist_object where " + "object_id in (select oid from pg_database where datname = '%s')",dbName); + return pg_dist_object_delete.data; +} List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, @@ -373,6 +382,8 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } + + ObjectAddress dbAddress = { 0 }; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); if (!IsObjectDistributed(&dbAddress)) @@ -381,6 +392,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, } UnmarkObjectDistributed(&dbAddress); + char *unmarkDatabaseDistributedSql = GetUnmarkDatabaseDistributedSql(stmt->dbname); char *dropDatabaseCommand = DeparseTreeNode(node); @@ -390,7 +402,8 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, quote_literal_cstr(dropDatabaseCommand)); - List *commands = list_make3(DISABLE_DDL_PROPAGATION, + List *commands = list_make4(DISABLE_DDL_PROPAGATION, + unmarkDatabaseDistributedSql, (void *) internalDropCommand->data, ENABLE_DDL_PROPAGATION); From ea0a908702da5d35f7d03b6f7378a1b1ee87ff28 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 23 Oct 2023 12:31:25 +0300 Subject: [PATCH 068/180] Fixes pg_dist_objects record in coordinators --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 7080eeba82a..444745b4cf2 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -357,7 +357,7 @@ static char * GetUnmarkDatabaseDistributedSql(char* dbName){ StringInfoData pg_dist_object_delete = { 0 }; initStringInfo(&pg_dist_object_delete); appendStringInfo(&pg_dist_object_delete, "delete from pg_dist_object where " - "object_id in (select oid from pg_database where datname = '%s')",dbName); + "objid in (select oid from pg_database where datname = '%s')",dbName); return pg_dist_object_delete.data; } From 690276c51621c15c6ecc2a25dfd184e663d7633b Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 23 Oct 2023 12:35:53 +0300 Subject: [PATCH 069/180] Fixes indentation --- src/backend/distributed/commands/database.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 444745b4cf2..88aea7abb64 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -241,7 +241,6 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, } - AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); EnsureCoordinator(); @@ -353,14 +352,19 @@ citus_internal_database_command(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } -static char * GetUnmarkDatabaseDistributedSql(char* dbName){ + +static char * +GetUnmarkDatabaseDistributedSql(char *dbName) +{ StringInfoData pg_dist_object_delete = { 0 }; initStringInfo(&pg_dist_object_delete); appendStringInfo(&pg_dist_object_delete, "delete from pg_dist_object where " - "objid in (select oid from pg_database where datname = '%s')",dbName); + "objid in (select oid from pg_database where datname = '%s')", + dbName); return pg_dist_object_delete.data; } + List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -383,7 +387,6 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, } - ObjectAddress dbAddress = { 0 }; ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); if (!IsObjectDistributed(&dbAddress)) From c9dae2684fdbbf3d91f740b0d5360542df74dd17 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 10:09:13 +0300 Subject: [PATCH 070/180] tests db as role --- src/backend/distributed/commands/common.c | 30 +++++++++++++++++++ src/backend/distributed/commands/database.c | 20 ++----------- src/backend/distributed/commands/role.c | 4 +-- .../distributed/commands/utility_hook.c | 20 +++++-------- src/include/distributed/commands.h | 3 ++ 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index 797981d47d3..bfc03cca176 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -28,6 +28,8 @@ #include "distributed/metadata/distobject.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" +#include "catalog/pg_database.h" +#include "commands/dbcommands.h" /* @@ -339,3 +341,31 @@ DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostproces return objectAddresses; } + +void UnmarkRolesAndDatabaseDistributed(Node *node) +{ + if (IsA(node, DropRoleStmt)) + { + DropRoleStmt *stmt = castNode(DropRoleStmt, node); + List *allDropRoles = stmt->roles; + + List *distributedDropRoles = FilterDistributedRoles(allDropRoles); + if (list_length(distributedDropRoles) > 0) + { + UnmarkRolesDistributed(distributedDropRoles); + } + + } + else if (IsA(node, DropdbStmt)) + { + elog(LOG, "Unmarking database1 as distributed"); + DropdbStmt *stmt = castNode(DropdbStmt, node); + char *dbName = stmt->dbname; + + Oid dbOid = get_database_oid(dbName, stmt->missing_ok); + ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbAddress, DatabaseRelationId, dbOid); + UnmarkObjectDistributed(dbAddress); + elog(LOG, "Unmarking database %s as distributed", dbName); + } +} diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 88aea7abb64..26626d5624b 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -352,19 +352,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } - -static char * -GetUnmarkDatabaseDistributedSql(char *dbName) -{ - StringInfoData pg_dist_object_delete = { 0 }; - initStringInfo(&pg_dist_object_delete); - appendStringInfo(&pg_dist_object_delete, "delete from pg_dist_object where " - "objid in (select oid from pg_database where datname = '%s')", - dbName); - return pg_dist_object_delete.data; -} - - List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -375,6 +362,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, } EnsureCoordinator(); + EnsureSequentialModeForRoleDDL(); DropdbStmt *stmt = (DropdbStmt *) node; @@ -394,9 +382,6 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } - UnmarkObjectDistributed(&dbAddress); - char *unmarkDatabaseDistributedSql = GetUnmarkDatabaseDistributedSql(stmt->dbname); - char *dropDatabaseCommand = DeparseTreeNode(node); StringInfo internalDropCommand = makeStringInfo(); @@ -405,8 +390,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, quote_literal_cstr(dropDatabaseCommand)); - List *commands = list_make4(DISABLE_DDL_PROPAGATION, - unmarkDatabaseDistributedSql, + List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) internalDropCommand->data, ENABLE_DDL_PROPAGATION); diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 754be1a2ba7..34be44637f3 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -65,7 +65,6 @@ static DefElem * makeDefElemBool(char *name, bool value); static List * GenerateRoleOptionsList(HeapTuple tuple); static List * GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options); static List * GenerateGrantRoleStmtsOfRole(Oid roleid); -static void EnsureSequentialModeForRoleDDL(void); static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); @@ -1080,6 +1079,7 @@ UnmarkRolesDistributed(List *roles) } ObjectAddressSet(roleAddress, AuthIdRelationId, roleOid); + elog(LOG, "Unmarking role %s as distributed", role->rolename); UnmarkObjectDistributed(&roleAddress); } } @@ -1278,7 +1278,7 @@ CreateRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) * with the role the role needs to be visible on all connections used by the transaction, * meaning we can only use 1 connection per node. */ -static void +void EnsureSequentialModeForRoleDDL(void) { if (!IsTransactionBlock()) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index dd729cad0ac..108f6b50a4e 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -80,6 +80,7 @@ #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/syscache.h" +#include "catalog/pg_database.h" bool EnableDDLPropagation = true; /* ddl propagation is enabled */ @@ -148,6 +149,7 @@ multi_ProcessUtility(PlannedStmt *pstmt, DestReceiver *dest, QueryCompletion *completionTag) { + elog(LOG, "multi_ProcessUtility called"); if (readOnlyTree) { pstmt = copyObject(pstmt); @@ -578,6 +580,8 @@ ProcessUtilityInternal(PlannedStmt *pstmt, PreprocessLockStatement((LockStmt *) parsetree, context); } + + /* * We only process ALTER TABLE ... ATTACH PARTITION commands in the function below * and distribute the partition if necessary. @@ -724,22 +728,12 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } /* - * Make sure that dropping the role deletes the pg_dist_object entries. There is a - * separate logic for roles, since roles are not included as dropped objects in the + * Make sure that dropping the role and database deletes the pg_dist_object entries. There is a + * separate logic for roles and database, since roles database are not included as dropped objects in the * drop event trigger. To handle it both on worker and coordinator nodes, it is not * implemented as a part of process functions but here. */ - if (IsA(parsetree, DropRoleStmt)) - { - DropRoleStmt *stmt = castNode(DropRoleStmt, parsetree); - List *allDropRoles = stmt->roles; - - List *distributedDropRoles = FilterDistributedRoles(allDropRoles); - if (list_length(distributedDropRoles) > 0) - { - UnmarkRolesDistributed(distributedDropRoles); - } - } + UnmarkRolesAndDatabaseDistributed(parsetree); pstmt->utilityStmt = parsetree; diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index b1f65177e35..cde0bd4f042 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -193,6 +193,7 @@ extern List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, boo isPostprocess); extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern void UnmarkRolesAndDatabaseDistributed(Node *node); /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); @@ -241,6 +242,7 @@ extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString) extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); + /* domain.c - forward declarations */ extern List * CreateDomainStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); @@ -510,6 +512,7 @@ extern List * RenameRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool extern void UnmarkRolesDistributed(List *roles); extern List * FilterDistributedRoles(List *roles); +extern void EnsureSequentialModeForRoleDDL(void); /* schema.c - forward declarations */ extern List * PostprocessCreateSchemaStmt(Node *node, const char *queryString); From 2d009d46d3042dd1500c31877901f190f1d47628 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 10:32:27 +0300 Subject: [PATCH 071/180] Fixes worker pg_dist_object removal issue --- src/backend/distributed/commands/database.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 88aea7abb64..94de7f5083d 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -365,6 +365,20 @@ GetUnmarkDatabaseDistributedSql(char *dbName) } +static void +UnmarkObjectDistributedForDropDb(const ObjectAddress *distAddress, char *dbName) +{ + UnmarkObjectDistributed(distAddress); + + if (EnableMetadataSync) + { + char *workerPgDistObjectUpdateCommand = + GetUnmarkDatabaseDistributedSql(dbName); + SendCommandToWorkersWithMetadata(workerPgDistObjectUpdateCommand); + } +} + + List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -394,7 +408,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NIL; } - UnmarkObjectDistributed(&dbAddress); + UnmarkObjectDistributedForDropDb(&dbAddress, stmt->dbname); char *unmarkDatabaseDistributedSql = GetUnmarkDatabaseDistributedSql(stmt->dbname); char *dropDatabaseCommand = DeparseTreeNode(node); From 73f0db2aeddd6c39dfd6aed88afa7261349cfad3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 14:09:36 +0300 Subject: [PATCH 072/180] Fixes create and drop database transaction use --- src/backend/distributed/commands/common.c | 27 ---------- src/backend/distributed/commands/database.c | 51 +++++++++---------- .../commands/distribute_object_ops.c | 2 +- .../distributed/commands/utility_hook.c | 22 +++++++- src/backend/distributed/metadata/distobject.c | 27 ++++++++++ src/include/distributed/commands.h | 3 +- .../distributed/commands/utility_hook.h | 1 + src/include/distributed/metadata/distobject.h | 1 + 8 files changed, 77 insertions(+), 57 deletions(-) diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index bfc03cca176..b338792d861 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -342,30 +342,3 @@ DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostproces return objectAddresses; } -void UnmarkRolesAndDatabaseDistributed(Node *node) -{ - if (IsA(node, DropRoleStmt)) - { - DropRoleStmt *stmt = castNode(DropRoleStmt, node); - List *allDropRoles = stmt->roles; - - List *distributedDropRoles = FilterDistributedRoles(allDropRoles); - if (list_length(distributedDropRoles) > 0) - { - UnmarkRolesDistributed(distributedDropRoles); - } - - } - else if (IsA(node, DropdbStmt)) - { - elog(LOG, "Unmarking database1 as distributed"); - DropdbStmt *stmt = castNode(DropdbStmt, node); - char *dbName = stmt->dbname; - - Oid dbOid = get_database_oid(dbName, stmt->missing_ok); - ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); - ObjectAddressSet(*dbAddress, DatabaseRelationId, dbOid); - UnmarkObjectDistributed(dbAddress); - elog(LOG, "Unmarking database %s as distributed", dbName); - } -} diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 26626d5624b..34813085e86 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -273,16 +273,11 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) char *createDatabaseCommand = DeparseTreeNode(node); - StringInfo internalCreateCommand = makeStringInfo(); - appendStringInfo(internalCreateCommand, - "SELECT pg_catalog.citus_internal_database_command(%s)", - quote_literal_cstr(createDatabaseCommand)); - List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *) internalCreateCommand->data, + (void *) createDatabaseCommand, ENABLE_DDL_PROPAGATION); - return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); + return NontransactionalNodeDDLTask(NON_COORDINATOR_NODES, commands); } @@ -356,55 +351,59 @@ List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { + bool isPostProcess = false; if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; } EnsureCoordinator(); - EnsureSequentialModeForRoleDDL(); DropdbStmt *stmt = (DropdbStmt *) node; - Oid databaseOid = get_database_oid(stmt->dbname, stmt->missing_ok); + List *addresses = GetObjectAddressListFromParseTree(node, stmt->missing_ok, isPostProcess); - if (databaseOid == InvalidOid) + if (list_length(addresses) == 0) { - /* let regular ProcessUtility deal with IF NOT EXISTS */ return NIL; } - - ObjectAddress dbAddress = { 0 }; - ObjectAddressSet(dbAddress, DatabaseRelationId, databaseOid); - if (!IsObjectDistributed(&dbAddress)) + ObjectAddress *address = (ObjectAddress *) linitial(addresses); + if (address->objectId == InvalidOid ||!IsObjectDistributed(address)) { return NIL; } char *dropDatabaseCommand = DeparseTreeNode(node); - StringInfo internalDropCommand = makeStringInfo(); - appendStringInfo(internalDropCommand, - "SELECT pg_catalog.citus_internal_database_command(%s)", - quote_literal_cstr(dropDatabaseCommand)); - List *commands = list_make3(DISABLE_DDL_PROPAGATION, - (void *) internalDropCommand->data, + (void *) dropDatabaseCommand, ENABLE_DDL_PROPAGATION); - return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); + return NontransactionalNodeDDLTask(NON_COORDINATOR_NODES, commands); } +static ObjectAddress *GetDatabaseAddressFromDatabaseName(char *databaseName) +{ + Oid databaseOid = get_database_oid(databaseName, false); + ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbAddress, DatabaseRelationId, databaseOid); + return dbAddress; +} + +List * +DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) +{ + DropdbStmt *stmt = castNode(DropdbStmt, node); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname); + return list_make1(dbAddress); +} List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); - Oid databaseOid = get_database_oid(stmt->dbname, missing_ok); - ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); - ObjectAddressSet(*dbAddress, DatabaseRelationId, databaseOid); - + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname); return list_make1(dbAddress); } diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 49a96e01653..2888bef1d13 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -484,7 +484,7 @@ static DistributeObjectOps Database_Drop = { .postprocess = NULL, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_DROP, - .address = NULL, + .address = DropDatabaseStmtObjectAddress, .markDistributed = false, }; diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 108f6b50a4e..3cd2ccd1ffa 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -729,7 +729,7 @@ ProcessUtilityInternal(PlannedStmt *pstmt, /* * Make sure that dropping the role and database deletes the pg_dist_object entries. There is a - * separate logic for roles and database, since roles database are not included as dropped objects in the + * separate logic for roles and database, since roles and database are not included as dropped objects in the * drop event trigger. To handle it both on worker and coordinator nodes, it is not * implemented as a part of process functions but here. */ @@ -1482,13 +1482,31 @@ DDLTaskList(Oid relationId, const char *commandString) return taskList; } +/* + * NontransactionalNodeDDLTask builds a list of tasks to execute a DDL command on a + * given target set of nodes with cannotBeExecutedInTransction is set to make sure + * that list is being executed without a transaction. + */ +List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands ){ + List *ddlJobs = NodeDDLTaskList(NON_COORDINATOR_NODES, commands); + DDLJob *ddlJob = NULL; + foreach_ptr(ddlJob, ddlJobs) + { + Task *task = NULL; + foreach_ptr(task, ddlJob->taskList) + { + task->cannotBeExecutedInTransction = true; + } + } + return ddlJobs; +} /* * NodeDDLTaskList builds a list of tasks to execute a DDL command on a * given target set of nodes. */ List * -NodeDDLTaskList(TargetWorkerSet targets, List *commands) +NodeDDLTaskList(TargetWorkerSet targets, List *commands ) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = InvalidObjectAddress; diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index c420e6ec390..722f51bc921 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -48,6 +48,8 @@ #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/rel.h" +#include "catalog/pg_database.h" +#include "commands/dbcommands.h" static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); @@ -355,6 +357,31 @@ ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, return spiStatus; } +void UnmarkRolesAndDatabaseDistributed(Node *node) +{ + if (IsA(node, DropRoleStmt)) + { + DropRoleStmt *stmt = castNode(DropRoleStmt, node); + List *allDropRoles = stmt->roles; + + List *distributedDropRoles = FilterDistributedRoles(allDropRoles); + if (list_length(distributedDropRoles) > 0) + { + UnmarkRolesDistributed(distributedDropRoles); + } + + } + else if (IsA(node, DropdbStmt)) + { + DropdbStmt *stmt = castNode(DropdbStmt, node); + char *dbName = stmt->dbname; + + Oid dbOid = get_database_oid(dbName, stmt->missing_ok); + ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbAddress, DatabaseRelationId, dbOid); + UnmarkObjectDistributed(dbAddress); + } +} /* * UnmarkObjectDistributed removes the entry from pg_dist_object that marks this object as diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index cde0bd4f042..95d6e9a1337 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -193,7 +193,6 @@ extern List * DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, boo isPostprocess); extern List * DropTextSearchDictObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern void UnmarkRolesAndDatabaseDistributed(Node *node); /* index.c */ typedef void (*PGIndexProcessor)(Form_pg_index, List **, int); @@ -241,6 +240,8 @@ extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); +extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); /* domain.c - forward declarations */ diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 9ae57b49aa1..3295d110c66 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -94,6 +94,7 @@ extern void ProcessUtilityParseTree(Node *node, const char *queryString, extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); extern List * DDLTaskList(Oid relationId, const char *commandString); +extern List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands ); extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands); extern bool AlterTableInProgress(void); extern bool DropSchemaOrDBInProgress(void); diff --git a/src/include/distributed/metadata/distobject.h b/src/include/distributed/metadata/distobject.h index ba984091c51..86fada5f742 100644 --- a/src/include/distributed/metadata/distobject.h +++ b/src/include/distributed/metadata/distobject.h @@ -27,6 +27,7 @@ extern void MarkObjectDistributed(const ObjectAddress *distAddress); extern void MarkObjectDistributedViaSuperUser(const ObjectAddress *distAddress); extern void MarkObjectDistributedLocally(const ObjectAddress *distAddress); extern void UnmarkObjectDistributed(const ObjectAddress *address); +extern void UnmarkRolesAndDatabaseDistributed(Node *node); extern bool IsTableOwnedByExtension(Oid relationId); extern bool ObjectAddressDependsOnExtension(const ObjectAddress *target); extern bool IsAnyObjectAddressOwnedByExtension(const List *targets, From e9e64a69c1dce29a5bd976cac0fe8238edbc6e07 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 14:18:10 +0300 Subject: [PATCH 073/180] Fixes indentation --- src/backend/distributed/commands/common.c | 1 - src/backend/distributed/commands/database.c | 12 ++++++--- .../distributed/commands/utility_hook.c | 27 ++++++++++--------- src/backend/distributed/metadata/distobject.c | 8 +++--- src/include/distributed/commands.h | 6 +++-- .../distributed/commands/utility_hook.h | 2 +- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index b338792d861..9a87df9f103 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -341,4 +341,3 @@ DropTextSearchConfigObjectAddress(Node *node, bool missing_ok, bool isPostproces return objectAddresses; } - diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 34813085e86..6522571faa7 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -347,6 +347,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -361,7 +362,8 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, DropdbStmt *stmt = (DropdbStmt *) node; - List *addresses = GetObjectAddressListFromParseTree(node, stmt->missing_ok, isPostProcess); + List *addresses = GetObjectAddressListFromParseTree(node, stmt->missing_ok, + isPostProcess); if (list_length(addresses) == 0) { @@ -369,7 +371,7 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, } ObjectAddress *address = (ObjectAddress *) linitial(addresses); - if (address->objectId == InvalidOid ||!IsObjectDistributed(address)) + if (address->objectId == InvalidOid || !IsObjectDistributed(address)) { return NIL; } @@ -384,7 +386,9 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, return NontransactionalNodeDDLTask(NON_COORDINATOR_NODES, commands); } -static ObjectAddress *GetDatabaseAddressFromDatabaseName(char *databaseName) + +static ObjectAddress * +GetDatabaseAddressFromDatabaseName(char *databaseName) { Oid databaseOid = get_database_oid(databaseName, false); ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); @@ -392,6 +396,7 @@ static ObjectAddress *GetDatabaseAddressFromDatabaseName(char *databaseName) return dbAddress; } + List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { @@ -400,6 +405,7 @@ DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) return list_make1(dbAddress); } + List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 3cd2ccd1ffa..b0fae67275f 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -581,7 +581,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt, } - /* * We only process ALTER TABLE ... ATTACH PARTITION commands in the function below * and distribute the partition if necessary. @@ -1482,31 +1481,35 @@ DDLTaskList(Oid relationId, const char *commandString) return taskList; } + /* * NontransactionalNodeDDLTask builds a list of tasks to execute a DDL command on a * given target set of nodes with cannotBeExecutedInTransction is set to make sure * that list is being executed without a transaction. */ -List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands ){ +List * +NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands) +{ List *ddlJobs = NodeDDLTaskList(NON_COORDINATOR_NODES, commands); - DDLJob *ddlJob = NULL; - foreach_ptr(ddlJob, ddlJobs) - { - Task *task = NULL; - foreach_ptr(task, ddlJob->taskList) - { - task->cannotBeExecutedInTransction = true; - } - } + DDLJob *ddlJob = NULL; + foreach_ptr(ddlJob, ddlJobs) + { + Task *task = NULL; + foreach_ptr(task, ddlJob->taskList) + { + task->cannotBeExecutedInTransction = true; + } + } return ddlJobs; } + /* * NodeDDLTaskList builds a list of tasks to execute a DDL command on a * given target set of nodes. */ List * -NodeDDLTaskList(TargetWorkerSet targets, List *commands ) +NodeDDLTaskList(TargetWorkerSet targets, List *commands) { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetObjectAddress = InvalidObjectAddress; diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 722f51bc921..af8354ee3b8 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -357,7 +357,9 @@ ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, return spiStatus; } -void UnmarkRolesAndDatabaseDistributed(Node *node) + +void +UnmarkRolesAndDatabaseDistributed(Node *node) { if (IsA(node, DropRoleStmt)) { @@ -369,12 +371,11 @@ void UnmarkRolesAndDatabaseDistributed(Node *node) { UnmarkRolesDistributed(distributedDropRoles); } - } else if (IsA(node, DropdbStmt)) { DropdbStmt *stmt = castNode(DropdbStmt, node); - char *dbName = stmt->dbname; + char *dbName = stmt->dbname; Oid dbOid = get_database_oid(dbName, stmt->missing_ok); ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); @@ -383,6 +384,7 @@ void UnmarkRolesAndDatabaseDistributed(Node *node) } } + /* * UnmarkObjectDistributed removes the entry from pg_dist_object that marks this object as * distributed. This will prevent updates to that object to be propagated to the worker. diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 95d6e9a1337..06dda53ebb7 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -240,8 +240,10 @@ extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); -extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); +extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool + isPostprocess); /* domain.c - forward declarations */ diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 3295d110c66..93d9e835550 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -94,7 +94,7 @@ extern void ProcessUtilityParseTree(Node *node, const char *queryString, extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); extern List * DDLTaskList(Oid relationId, const char *commandString); -extern List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands ); +extern List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands); extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands); extern bool AlterTableInProgress(void); extern bool DropSchemaOrDBInProgress(void); From fb9f75bb241cf196c612241f4a26cada03fd37e4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 14:43:12 +0300 Subject: [PATCH 074/180] Removes unnecessary logs --- src/backend/distributed/commands/utility_hook.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index b0fae67275f..27b2b9ddca6 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -149,7 +149,6 @@ multi_ProcessUtility(PlannedStmt *pstmt, DestReceiver *dest, QueryCompletion *completionTag) { - elog(LOG, "multi_ProcessUtility called"); if (readOnlyTree) { pstmt = copyObject(pstmt); From 1dcee370e11421f6232c314799804329b8f8d4df Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 14:44:34 +0300 Subject: [PATCH 075/180] Fixes create_drop_database_test --- .../regress/expected/create_drop_database_propagation.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 32552d01ab7..088a445d6cc 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -107,15 +107,15 @@ set citus.enable_create_database_propagation=on; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE DATABASE%'; create database "mydatabase#1'2"; -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE "mydatabase#1''2"') +NOTICE: issuing CREATE DATABASE "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE "mydatabase#1''2"') +NOTICE: issuing CREATE DATABASE "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx set citus.grep_remote_commands = '%DROP DATABASE%'; drop database if exists "mydatabase#1'2"; -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE IF EXISTS "mydatabase#1''2"') +NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT pg_catalog.citus_internal_database_command('DROP DATABASE IF EXISTS "mydatabase#1''2"') +NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx --clean up resources created by this test drop tablespace create_drop_db_tablespace; From f779947b2b3522cf951497fbcea787cb9f08f15d Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 14:57:36 +0300 Subject: [PATCH 076/180] Removes unnecessary logs --- src/backend/distributed/commands/role.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 34be44637f3..792efd93494 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -1079,7 +1079,6 @@ UnmarkRolesDistributed(List *roles) } ObjectAddressSet(roleAddress, AuthIdRelationId, roleOid); - elog(LOG, "Unmarking role %s as distributed", role->rolename); UnmarkObjectDistributed(&roleAddress); } } From b5cbc048b70eef8fd4072943d6b4bce6218392b0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 15:12:08 +0300 Subject: [PATCH 077/180] Fixes missing ok issue --- src/backend/distributed/commands/database.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 6522571faa7..1ba0a1639a7 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -388,9 +388,9 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, static ObjectAddress * -GetDatabaseAddressFromDatabaseName(char *databaseName) +GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) { - Oid databaseOid = get_database_oid(databaseName, false); + Oid databaseOid = get_database_oid(databaseName, missingOk); ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*dbAddress, DatabaseRelationId, databaseOid); return dbAddress; @@ -401,7 +401,7 @@ List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropdbStmt *stmt = castNode(DropdbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, stmt->missing_ok); return list_make1(dbAddress); } @@ -410,6 +410,6 @@ List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname,stmt->missing_ok); return list_make1(dbAddress); } From 3a0a5ae3449e0157138e98ddf4d13423786dcc12 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 15:13:22 +0300 Subject: [PATCH 078/180] Fixes missin ok issue --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 1ba0a1639a7..d6c02d49248 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -410,6 +410,6 @@ List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname,stmt->missing_ok); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname,missing_ok); return list_make1(dbAddress); } From 2f52fbd13ea2b4600b1fd49900e8661303b5192d Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 15:14:05 +0300 Subject: [PATCH 079/180] Fixes missing ok issue --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index d6c02d49248..8d2b14f6103 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -401,7 +401,7 @@ List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropdbStmt *stmt = castNode(DropdbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, stmt->missing_ok); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missing_ok); return list_make1(dbAddress); } From c7b040c3cdb16998f03b0036f7d0fbd71e00f9a2 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 15:19:50 +0300 Subject: [PATCH 080/180] Fixes indentation --- src/backend/distributed/commands/database.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 8d2b14f6103..7f0f03aa692 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -401,7 +401,8 @@ List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { DropdbStmt *stmt = castNode(DropdbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, missing_ok); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, + missing_ok); return list_make1(dbAddress); } @@ -410,6 +411,7 @@ List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname,missing_ok); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, + missing_ok); return list_make1(dbAddress); } From c7da2cd122c4870e6e371bdcabf05086604e0230 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 15:44:59 +0300 Subject: [PATCH 081/180] Fixes cannotBeExecutedInTransction --- src/backend/distributed/commands/utility_hook.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index d5235290707..28159b6169a 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -1483,7 +1483,7 @@ DDLTaskList(Oid relationId, const char *commandString) /* * NontransactionalNodeDDLTask builds a list of tasks to execute a DDL command on a - * given target set of nodes with cannotBeExecutedInTransction is set to make sure + * given target set of nodes with cannotBeExecutedInTransaction is set to make sure * that list is being executed without a transaction. */ List * @@ -1496,7 +1496,7 @@ NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands) Task *task = NULL; foreach_ptr(task, ddlJob->taskList) { - task->cannotBeExecutedInTransction = true; + task->cannotBeExecutedInTransaction = true; } } return ddlJobs; From 48d749e2e8daceeb3f13d2165d9b6684fc548898 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 19:15:46 +0300 Subject: [PATCH 082/180] Fixes message for non-transaction-safe commands --- src/backend/distributed/commands/utility_hook.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 28159b6169a..85ae27356a8 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -1265,9 +1265,10 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob) { ereport(WARNING, (errmsg( - "CONCURRENTLY-enabled index commands can fail partially, " - "leaving behind an INVALID index.\n Use DROP INDEX " - "CONCURRENTLY IF EXISTS to remove the invalid index."))); + "Commands that are not transaction-safe may result in partial failure" + ", potentially leading to an inconsistent state. If the problematic command" + " is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the " + "object, if applicable, and then reattempt the original command."))); PG_RE_THROW(); } } From c437850db5d60868f7814e0cb1bed1579464e4f8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 19:25:26 +0300 Subject: [PATCH 083/180] Beatufies error message --- src/backend/distributed/commands/utility_hook.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 85ae27356a8..fc763f369ad 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -1266,9 +1266,9 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob) ereport(WARNING, (errmsg( "Commands that are not transaction-safe may result in partial failure" - ", potentially leading to an inconsistent state. If the problematic command" + ", potentially leading to an inconsistent state.\nIf the problematic command" " is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the " - "object, if applicable, and then reattempt the original command."))); + "object,\nif applicable, and then reattempt the original command."))); PG_RE_THROW(); } } From c8fcf080c261e99484dd164fa2b0c47be384a200 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 24 Oct 2023 20:44:43 +0300 Subject: [PATCH 084/180] Fixes out files for transaction message --- .../failure_create_index_concurrently.out | 30 +++++++++++-------- src/test/regress/expected/single_node.out | 5 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/test/regress/expected/failure_create_index_concurrently.out b/src/test/regress/expected/failure_create_index_concurrently.out index a198ddc7087..94d0f373dc1 100644 --- a/src/test/regress/expected/failure_create_index_concurrently.out +++ b/src/test/regress/expected/failure_create_index_concurrently.out @@ -26,8 +26,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); (1 row) CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -59,8 +60,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); (1 row) CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -86,8 +88,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( (1 row) CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -111,8 +114,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( (1 row) CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -137,8 +141,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()'); (1 row) DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -164,8 +169,9 @@ SELECT create_distributed_table('index_test_2', 'a'); INSERT INTO index_test_2 VALUES (1, 1), (1, 2); CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. -Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: could not create unique index "index_test_2_a_idx_1880019" DETAIL: Key (a)=(1) is duplicated. CONTEXT: while executing command on localhost:xxxxx diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index f485763c5be..3b24fd5f55f 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -88,8 +88,9 @@ SELECT create_distributed_table('failover_to_local', 'a', shard_count=>32); (1 row) CREATE INDEX CONCURRENTLY ON failover_to_local(a); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. - Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: the total number of connections on the server is more than max_connections(100) HINT: Consider using a higher value for max_connections -- reset global GUC changes From 641e4136762de8ca092646aadf36096890f68587 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 26 Oct 2023 20:36:43 +0300 Subject: [PATCH 085/180] Adds metadatasync for create database --- src/backend/distributed/commands/database.c | 251 ++++++++++++++++++ .../distributed/metadata/metadata_sync.c | 7 + src/include/distributed/commands.h | 1 + 3 files changed, 259 insertions(+) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 7f0f03aa692..443c0f36631 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -35,6 +35,15 @@ #include "distributed/deparse_shard_query.h" #include "distributed/listutils.h" #include "distributed/adaptive_executor.h" +#include "access/htup_details.h" +#include "catalog/pg_tablespace.h" +#include "access/heapam.h" +#include "utils/relcache.h" +#include "utils/rel.h" +#include "utils/lsyscache.h" +#include "catalog/pg_collation.h" +#include "utils/relcache.h" +#include "catalog/pg_database_d.h" static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); @@ -415,3 +424,245 @@ CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) missing_ok); return list_make1(dbAddress); } + + +static char * +GetTablespaceName(Oid tablespaceOid) +{ + HeapTuple tuple = SearchSysCache1(TABLESPACEOID, ObjectIdGetDatum(tablespaceOid)); + if (!HeapTupleIsValid(tuple)) + { + return NULL; + } + + Form_pg_tablespace tablespaceForm = (Form_pg_tablespace) GETSTRUCT(tuple); + char *tablespaceName = NameStr(tablespaceForm->spcname); + + ReleaseSysCache(tuple); + + return tablespaceName; +} + + +/* + * DatabaseCollationInfo is used to store collation related information of a database + */ +typedef struct DatabaseCollationInfo +{ + char *collation; + char *ctype; + char *icu_locale; + char *collversion; +} DatabaseCollationInfo; + +/* + * GetDatabaseCollation gets oid of a database and returns all the collation related information + * We need this method since collation related info in Form_pg_database is not accessible + */ +static DatabaseCollationInfo +GetDatabaseCollation(Oid db_oid) +{ + HeapTuple tup; + DatabaseCollationInfo info; + Datum collationDatum, ctypeDatum, icuLocaleDatum, collverDatum; + bool isNull; + Relation rel; + TupleDesc tupdesc; + Snapshot snapshot; + + snapshot = RegisterSnapshot(GetLatestSnapshot()); + rel = table_open(DatabaseRelationId, AccessShareLock); + tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, db_oid); + if (!HeapTupleIsValid(tup)) + { + elog(ERROR, "cache lookup failed for database %u", db_oid); + } + + tupdesc = RelationGetDescr(rel); + collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); + if (isNull) + { + info.collation = NULL; + } + else + { + info.collation = TextDatumGetCString(collationDatum); + } + + ctypeDatum = heap_getattr(tup, Anum_pg_database_datctype, tupdesc, &isNull); + if (isNull) + { + info.ctype = NULL; + } + else + { + info.ctype = TextDatumGetCString(ctypeDatum); + } + + icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); + if (isNull) + { + info.icu_locale = NULL; + } + else + { + info.icu_locale = TextDatumGetCString(icuLocaleDatum); + } + + collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); + if (isNull) + { + info.collversion = NULL; + } + else + { + info.collversion = TextDatumGetCString(collverDatum); + } + + table_close(rel, AccessShareLock); + UnregisterSnapshot(snapshot); + heap_freetuple(tup); + + return info; +} + + +static void +FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) +{ + if (collInfo.collation != NULL) + { + pfree(collInfo.collation); + } + if (collInfo.ctype != NULL) + { + pfree(collInfo.ctype); + } + if (collInfo.icu_locale != NULL) + { + pfree(collInfo.icu_locale); + } +} + + +/* + * GenerateCreateDatabaseStatementFromPgDatabase is gets the pg_database tuple and returns the CREATE DATABASE statement + */ +static char * +GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) +{ + DatabaseCollationInfo collInfo = GetDatabaseCollation(databaseForm->oid); + elog(LOG, "collInfo: %s %s %s %s", collInfo.collation, collInfo.ctype, + collInfo.icu_locale, collInfo.collversion); + + StringInfoData str; + initStringInfo(&str); + + appendStringInfo(&str, "CREATE DATABASE %s", quote_identifier(NameStr( + databaseForm-> + datname))); + + if (databaseForm->datdba != InvalidOid) + { + appendStringInfo(&str, " OWNER = %s", GetUserNameFromId(databaseForm->datdba, + false)); + } + + if (databaseForm->encoding != -1) + { + appendStringInfo(&str, " ENCODING = '%s'", pg_encoding_to_char( + databaseForm->encoding)); + } + + if (collInfo.collation != NULL) + { + appendStringInfo(&str, " LC_COLLATE = '%s'", collInfo.collation); + } + if (collInfo.ctype != NULL) + { + appendStringInfo(&str, " LC_CTYPE = '%s'", collInfo.ctype); + } + + if (collInfo.icu_locale != NULL) + { + appendStringInfo(&str, " ICU_LOCALE = '%s'", collInfo.icu_locale); + } + + if (databaseForm->datlocprovider != 0) + { + appendStringInfo(&str, " LOCALE_PROVIDER = '%c'", databaseForm->datlocprovider); + } + + if (collInfo.collversion != NULL) + { + appendStringInfo(&str, " COLLATION_VERSION = '%s'", collInfo.collversion); + } + + if (databaseForm->dattablespace != InvalidOid) + { + appendStringInfo(&str, " TABLESPACE = %s", quote_identifier(GetTablespaceName( + databaseForm-> + dattablespace))); + } + + appendStringInfo(&str, " ALLOW_CONNECTIONS = '%s'", databaseForm->datallowconn ? + "true" : "false"); + + if (databaseForm->datconnlimit >= 0) + { + appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); + } + + appendStringInfo(&str, " IS_TEMPLATE = '%s'", databaseForm->datistemplate ? "true" : + "false"); + + FreeDatabaseCollationInfo(collInfo); + + + return str.data; +} + + +/* + * GenerateCreateDatabaseCommandList is gets the pg_database tuples and returns the CREATE DATABASE statement list + * for all the databases in the cluster.citus_internal_database_command UDF is used to send the CREATE DATABASE + * statement to the workers since the CREATE DATABASE statement gives error in transaction context. + */ +List * +GenerateCreateDatabaseCommandList(void) +{ + List *commands = NIL; + HeapTuple tuple; + Relation pgDatabaseRel; + TableScanDesc scan; + + pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); + scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); + + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); + + char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); + + + StringInfo outerDbStmt; + outerDbStmt = makeStringInfo(); + + /* Generate the CREATE DATABASE statement */ + appendStringInfo(outerDbStmt, + "select pg_catalog.citus_internal_database_command( %s)", + quote_literal_cstr( + createStmt)); + + elog(LOG, "outerDbStmt: %s", outerDbStmt->data); + + /* Add the statement to the list of commands */ + commands = lappend(commands, outerDbStmt->data); + } + + heap_endscan(scan); + table_close(pgDatabaseRel, AccessShareLock); + + return commands; +} diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 40bdae0eaf2..54fa801ae89 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4501,6 +4501,13 @@ PropagateNodeWideObjectsCommandList(void) /* collect all commands */ List *ddlCommands = NIL; + if (EnableCreateDatabasePropagation) + { + /* Get commands for database creation */ + List *createDatabaseCommands = GenerateCreateDatabaseCommandList(); + ddlCommands = list_concat(ddlCommands, createDatabaseCommands); + } + if (EnableAlterRoleSetPropagation) { /* diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 06dda53ebb7..a4f890f9ea3 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -244,6 +244,7 @@ extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess); +extern List * GenerateCreateDatabaseCommandList(void); /* domain.c - forward declarations */ From 473f6cbf05b4f8e0e7cc8383a79d24e9e4c54ade Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 08:37:46 +0300 Subject: [PATCH 086/180] Fixes metadata sync --- src/backend/distributed/commands/database.c | 21 +- .../distributed/commands/utility_hook.c | 2 +- .../create_drop_database_propagation.out | 194 +++++++----- src/test/regress/expected/pg15.out | 292 +++++++++++++----- .../sql/create_drop_database_propagation.sql | 157 ++++++---- src/test/regress/sql/pg15.sql | 226 ++++++++++---- 6 files changed, 632 insertions(+), 260 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 443c0f36631..e7a8299ae8b 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -545,6 +545,23 @@ FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) } + +static char *get_locale_provider_string(char datlocprovider) +{ + switch (datlocprovider) + { + case 'c': + return "libc"; + case 'i': + return "icu"; + case 'l': + return "locale"; + default: + return ""; + } +} + + /* * GenerateCreateDatabaseStatementFromPgDatabase is gets the pg_database tuple and returns the CREATE DATABASE statement */ @@ -552,8 +569,6 @@ static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) { DatabaseCollationInfo collInfo = GetDatabaseCollation(databaseForm->oid); - elog(LOG, "collInfo: %s %s %s %s", collInfo.collation, collInfo.ctype, - collInfo.icu_locale, collInfo.collversion); StringInfoData str; initStringInfo(&str); @@ -590,7 +605,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) if (databaseForm->datlocprovider != 0) { - appendStringInfo(&str, " LOCALE_PROVIDER = '%c'", databaseForm->datlocprovider); + appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", get_locale_provider_string(databaseForm->datlocprovider)); } if (collInfo.collversion != NULL) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index fc763f369ad..0d400d1390c 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -1490,7 +1490,7 @@ DDLTaskList(Oid relationId, const char *commandString) List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands) { - List *ddlJobs = NodeDDLTaskList(NON_COORDINATOR_NODES, commands); + List *ddlJobs = NodeDDLTaskList(targets, commands); DDLJob *ddlJob = NULL; foreach_ptr(ddlJob, ddlJobs) { diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 088a445d6cc..78ce0025ef5 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -19,89 +19,143 @@ CREATE DATABASE mydatabase TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | tablespace +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_tablespace -(1 row) + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +drop database mydatabase; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace + + +(3 rows) + +-- test database syncing after node addition +select citus_remove_node('localhost', :worker_2_port); + citus_remove_node --------------------------------------------------------------------- - mydatabase | 6 | f | t | 10 | C | C | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace + (1 row) -\c - - - :master_port -set citus.enable_create_database_propagation=on; -drop database mydatabase; -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +--test with is_template true and allow connections false +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- -(0 rows) + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(2 rows) -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +select citus_add_node('localhost', :worker_2_port); + citus_add_node --------------------------------------------------------------------- -(0 rows) + 30 +(1 row) -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datistemplate | datallowconn | datconnlimit | datcollate | datctype | datacl | database_owner | database_owner | tablespace +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- -(0 rows) + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase; +NOTICE: issuing DROP DATABASE mydatabase +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE mydatabase +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + +(3 rows) -\c - - - :master_port --tests for special characters in database name set citus.enable_create_database_propagation=on; SET citus.log_remote_commands = true; diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index caee521a745..3ee5a15cf6e 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1548,6 +1548,8 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user @@ -1564,87 +1566,231 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false OID = 966345; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) +NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace -(1 row) +drop database mydatabase; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + +(3 rows) + +select citus_remove_node('localhost', :worker_2_port); + citus_remove_node +--------------------------------------------------------------------- -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace ---------------------------------------------------------------------- - mydatabase | 6 | i | f | t | 10 | C | C | und | 1.0 | | create_drop_db_test_user | create_drop_db_test_user | create_drop_db_tablespace (1 row) -\c - - - :master_port -set citus.enable_create_database_propagation=on; -drop database mydatabase; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +CREATE DATABASE mydatabase2 + WITH OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(2 rows) + +SET citus.log_remote_commands = true; +select citus_add_node('localhost', :worker_2_port); +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';select pg_catalog.citus_internal_database_command( 'CREATE DATABASE postgres OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE regression OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template1 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template0 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''false'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE mydatabase2 OWNER = create_drop_db_test_user ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' COLLATION_VERSION = ''1.0'' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');ALTER ROLE ALL IN DATABASE regression SET lc_messages = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_monetary = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_numeric = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_time = 'C';ALTER ROLE ALL IN DATABASE regression SET bytea_output = 'hex';ALTER ROLE ALL IN DATABASE regression SET timezone_abbreviations = 'Default';SET citus.enable_ddl_propagation TO 'on' + citus_add_node --------------------------------------------------------------------- -(0 rows) + 30 +(1 row) -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase2; +NOTICE: issuing DROP DATABASE mydatabase2 +NOTICE: issuing DROP DATABASE mydatabase2 +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- -(0 rows) -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; - datname | encoding | datlocprovider | datistemplate | datallowconn | datconnlimit | datcollate | datctype | daticulocale | datcollversion | datacl | database_owner | database_owner | tablespace + + +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; +NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + UPDATE 1 + UPDATE 1 + UPDATE 1 +(3 rows) + +; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; +NOTICE: issuing DROP DATABASE my_template_database +NOTICE: issuing DROP DATABASE my_template_database +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result --------------------------------------------------------------------- -(0 rows) + + + +(3 rows) \c - - - :master_port drop tablespace create_drop_db_tablespace; @@ -1653,4 +1799,4 @@ drop tablespace create_drop_db_tablespace; \c - - - :worker_2_port drop tablespace create_drop_db_tablespace; \c - - - :master_port -drop user create_drop_db_test_user; \ No newline at end of file +drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index d2195659570..6f9202a4ba0 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -16,83 +16,124 @@ create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; + CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 + WITH OWNER = create_drop_db_test_user + TEMPLATE = 'template0' ENCODING = 'UTF8' + CONNECTION LIMIT = 10 LC_COLLATE = 'C' LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; -\c - - - :worker_2_port +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; -\c - - - :master_port -set citus.enable_create_database_propagation=on; drop database mydatabase; -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; -\c - - - :worker_2_port +-- test database syncing after node addition -SELECT pd.datname, pd.encoding, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.datacl, rolname AS database_owner, -pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +select citus_remove_node('localhost', :worker_2_port); -\c - - - :master_port +--test with is_template true and allow connections false +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_COLLATE = 'C' + LC_CTYPE = 'C' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; + + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + +select citus_add_node('localhost', :worker_2_port); + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; --tests for special characters in database name set citus.enable_create_database_propagation=on; diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index f0c6706d2b2..ac523b52159 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -995,6 +995,8 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user @@ -1012,70 +1014,184 @@ CREATE DATABASE mydatabase IS_TEMPLATE = false OID = 966345; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +drop database mydatabase; -\c - - - :worker_2_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; -\c - - - :master_port -set citus.enable_create_database_propagation=on; -drop database mydatabase; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +select citus_remove_node('localhost', :worker_2_port); -\c - - - :worker_1_port -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; -\c - - - :worker_2_port +CREATE DATABASE mydatabase2 + WITH OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + + +SET citus.log_remote_commands = true; +select citus_add_node('localhost', :worker_2_port); + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase2; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + + + +SET citus.log_remote_commands = true; + +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + +; + +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; -SELECT pd.datname, pd.encoding, pd.datlocprovider, -pd.datistemplate, pd.datallowconn, pd.datconnlimit, -pd.datcollate , pd. datctype , pd.daticulocale, pd.datcollversion, -pd.datacl, rolname AS database_owner, pa.rolname AS database_owner, pt.spcname AS tablespace -FROM pg_database pd -JOIN pg_authid pa ON pd.datdba = pa.oid -join pg_tablespace pt on pd.dattablespace = pt.oid -WHERE datname = 'mydatabase'; \c - - - :master_port drop tablespace create_drop_db_tablespace; From a797584f1c2b53cf5d2254a3d446ad4bfd0286bd Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 09:07:19 +0300 Subject: [PATCH 087/180] Fixes tests --- src/backend/distributed/commands/database.c | 38 ++++++++++++------- .../create_drop_database_propagation.out | 6 +-- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index e7a8299ae8b..4e9528ffef7 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -545,20 +545,29 @@ FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) } - -static char *get_locale_provider_string(char datlocprovider) +static char * +get_locale_provider_string(char datlocprovider) { - switch (datlocprovider) - { - case 'c': - return "libc"; - case 'i': - return "icu"; - case 'l': - return "locale"; - default: - return ""; - } + switch (datlocprovider) + { + case 'c': + { + return "libc"; + } + + case 'i': + { + return "icu"; + } + + case 'l': + { + return "locale"; + } + + default: + return ""; + } } @@ -605,7 +614,8 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) if (databaseForm->datlocprovider != 0) { - appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", get_locale_provider_string(databaseForm->datlocprovider)); + appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", get_locale_provider_string( + databaseForm->datlocprovider)); } if (collInfo.collversion != NULL) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 78ce0025ef5..e05616a425d 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -10,10 +10,10 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 + WITH OWNER = create_drop_db_test_user + TEMPLATE = 'template0' ENCODING = 'UTF8' + CONNECTION LIMIT = 10 LC_COLLATE = 'C' LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace From 569f4db60c0c8044c2bc8af70bb7465d1a81b4e3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 09:51:15 +0300 Subject: [PATCH 088/180] Fixes pg 14 compile warnings --- src/backend/distributed/commands/database.c | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 4e9528ffef7..cb32231f1d6 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -451,8 +451,10 @@ typedef struct DatabaseCollationInfo { char *collation; char *ctype; + #if PG_VERSION_NUM >= PG_VERSION_15 char *icu_locale; char *collversion; + #endif } DatabaseCollationInfo; /* @@ -462,17 +464,15 @@ typedef struct DatabaseCollationInfo static DatabaseCollationInfo GetDatabaseCollation(Oid db_oid) { - HeapTuple tup; DatabaseCollationInfo info; Datum collationDatum, ctypeDatum, icuLocaleDatum, collverDatum; bool isNull; - Relation rel; TupleDesc tupdesc; Snapshot snapshot; snapshot = RegisterSnapshot(GetLatestSnapshot()); - rel = table_open(DatabaseRelationId, AccessShareLock); - tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, db_oid); + Relation rel = table_open(DatabaseRelationId, AccessShareLock); + HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, db_oid); if (!HeapTupleIsValid(tup)) { elog(ERROR, "cache lookup failed for database %u", db_oid); @@ -499,6 +499,8 @@ GetDatabaseCollation(Oid db_oid) info.ctype = TextDatumGetCString(ctypeDatum); } + #if PG_VERSION_NUM >= PG_VERSION_15 + icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); if (isNull) { @@ -518,6 +520,7 @@ GetDatabaseCollation(Oid db_oid) { info.collversion = TextDatumGetCString(collverDatum); } + #endif table_close(rel, AccessShareLock); UnregisterSnapshot(snapshot); @@ -607,6 +610,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " LC_CTYPE = '%s'", collInfo.ctype); } + #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.icu_locale != NULL) { appendStringInfo(&str, " ICU_LOCALE = '%s'", collInfo.icu_locale); @@ -622,6 +626,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) { appendStringInfo(&str, " COLLATION_VERSION = '%s'", collInfo.collversion); } + #endif if (databaseForm->dattablespace != InvalidOid) { @@ -658,11 +663,9 @@ GenerateCreateDatabaseCommandList(void) { List *commands = NIL; HeapTuple tuple; - Relation pgDatabaseRel; - TableScanDesc scan; - pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); - scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); + Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); + TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { @@ -671,8 +674,7 @@ GenerateCreateDatabaseCommandList(void) char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); - StringInfo outerDbStmt; - outerDbStmt = makeStringInfo(); + StringInfo outerDbStmt = makeStringInfo(); /* Generate the CREATE DATABASE statement */ appendStringInfo(outerDbStmt, From fe26aebf6cb0f50cc25f206d948f722fbbacf23a Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 09:52:58 +0300 Subject: [PATCH 089/180] Fixes useless declarations --- src/backend/distributed/commands/database.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index cb32231f1d6..33b8f6a51ff 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -467,10 +467,8 @@ GetDatabaseCollation(Oid db_oid) DatabaseCollationInfo info; Datum collationDatum, ctypeDatum, icuLocaleDatum, collverDatum; bool isNull; - TupleDesc tupdesc; - Snapshot snapshot; - snapshot = RegisterSnapshot(GetLatestSnapshot()); + Snapshot snapshot = RegisterSnapshot(GetLatestSnapshot()); Relation rel = table_open(DatabaseRelationId, AccessShareLock); HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, db_oid); if (!HeapTupleIsValid(tup)) @@ -478,7 +476,7 @@ GetDatabaseCollation(Oid db_oid) elog(ERROR, "cache lookup failed for database %u", db_oid); } - tupdesc = RelationGetDescr(rel); + TupleDesc tupdesc = RelationGetDescr(rel); collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); if (isNull) { From 2a6c40d6436a51189afbd55d007756bf3ee86c4c Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 10:10:42 +0300 Subject: [PATCH 090/180] Fixes flaky tests --- .../expected/create_drop_database_propagation.out | 12 ++++++------ src/test/regress/expected/pg15.out | 12 ++++++------ .../regress/sql/create_drop_database_propagation.sql | 4 ++-- src/test/regress/sql/pg15.sql | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index e05616a425d..9f98c740c4e 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -63,10 +63,10 @@ SELECT result from run_command_on_all_nodes( (3 rows) -- test database syncing after node addition -select citus_remove_node('localhost', :worker_2_port); - citus_remove_node +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? --------------------------------------------------------------------- - + 1 (1 row) --test with is_template true and allow connections false @@ -100,10 +100,10 @@ SELECT result from run_command_on_all_nodes( [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] (2 rows) -select citus_add_node('localhost', :worker_2_port); - citus_add_node +select 1 from citus_add_node('localhost', :worker_2_port); + ?column? --------------------------------------------------------------------- - 30 + 1 (1 row) SELECT result from run_command_on_all_nodes( diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index 3ee5a15cf6e..e58387551e8 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1611,10 +1611,10 @@ SELECT result from run_command_on_all_nodes( (3 rows) -select citus_remove_node('localhost', :worker_2_port); - citus_remove_node +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? --------------------------------------------------------------------- - + 1 (1 row) SET citus.log_remote_commands = true; @@ -1654,11 +1654,11 @@ SELECT result from run_command_on_all_nodes( (2 rows) SET citus.log_remote_commands = true; -select citus_add_node('localhost', :worker_2_port); +select 1 from citus_add_node('localhost', :worker_2_port); NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';select pg_catalog.citus_internal_database_command( 'CREATE DATABASE postgres OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE regression OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template1 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template0 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''false'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE mydatabase2 OWNER = create_drop_db_test_user ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' COLLATION_VERSION = ''1.0'' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');ALTER ROLE ALL IN DATABASE regression SET lc_messages = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_monetary = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_numeric = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_time = 'C';ALTER ROLE ALL IN DATABASE regression SET bytea_output = 'hex';ALTER ROLE ALL IN DATABASE regression SET timezone_abbreviations = 'Default';SET citus.enable_ddl_propagation TO 'on' - citus_add_node + ?column? --------------------------------------------------------------------- - 30 + 1 (1 row) SET citus.log_remote_commands = false; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 6f9202a4ba0..6b95fd83ea8 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -67,7 +67,7 @@ SELECT result from run_command_on_all_nodes( -- test database syncing after node addition -select citus_remove_node('localhost', :worker_2_port); +select 1 from citus_remove_node('localhost', :worker_2_port); --test with is_template true and allow connections false CREATE DATABASE mydatabase @@ -97,7 +97,7 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -select citus_add_node('localhost', :worker_2_port); +select 1 from citus_add_node('localhost', :worker_2_port); SELECT result from run_command_on_all_nodes( $$ diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index ac523b52159..845d5fab593 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1048,7 +1048,7 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; -select citus_remove_node('localhost', :worker_2_port); +select 1 from citus_remove_node('localhost', :worker_2_port); SET citus.log_remote_commands = true; @@ -1085,7 +1085,7 @@ SELECT result from run_command_on_all_nodes( SET citus.log_remote_commands = true; -select citus_add_node('localhost', :worker_2_port); +select 1 from citus_add_node('localhost', :worker_2_port); SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( From 8811d910a5472fddf2bd1e4f7f40f1e0ff9761b5 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 10:28:09 +0300 Subject: [PATCH 091/180] Fixes pg 14 compile errors --- src/backend/distributed/commands/database.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 33b8f6a51ff..c33afb77e52 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -465,7 +465,6 @@ static DatabaseCollationInfo GetDatabaseCollation(Oid db_oid) { DatabaseCollationInfo info; - Datum collationDatum, ctypeDatum, icuLocaleDatum, collverDatum; bool isNull; Snapshot snapshot = RegisterSnapshot(GetLatestSnapshot()); @@ -477,7 +476,7 @@ GetDatabaseCollation(Oid db_oid) } TupleDesc tupdesc = RelationGetDescr(rel); - collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); + Datum collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); if (isNull) { info.collation = NULL; @@ -487,7 +486,7 @@ GetDatabaseCollation(Oid db_oid) info.collation = TextDatumGetCString(collationDatum); } - ctypeDatum = heap_getattr(tup, Anum_pg_database_datctype, tupdesc, &isNull); + Datum ctypeDatum = heap_getattr(tup, Anum_pg_database_datctype, tupdesc, &isNull); if (isNull) { info.ctype = NULL; @@ -499,7 +498,7 @@ GetDatabaseCollation(Oid db_oid) #if PG_VERSION_NUM >= PG_VERSION_15 - icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); + Datum icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); if (isNull) { info.icu_locale = NULL; @@ -509,7 +508,7 @@ GetDatabaseCollation(Oid db_oid) info.icu_locale = TextDatumGetCString(icuLocaleDatum); } - collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); + Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); if (isNull) { info.collversion = NULL; @@ -539,13 +538,15 @@ FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) { pfree(collInfo.ctype); } + #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.icu_locale != NULL) { pfree(collInfo.icu_locale); } + #endif } - +#if PG_VERSION_NUM >= PG_VERSION_15 static char * get_locale_provider_string(char datlocprovider) { @@ -570,6 +571,7 @@ get_locale_provider_string(char datlocprovider) return ""; } } +#endif /* From b41feadd97d3c7375a17bbe0b56dd43a8ae8dd86 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 10:39:16 +0300 Subject: [PATCH 092/180] Fixes indent --- src/backend/distributed/commands/database.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index c33afb77e52..42ef2ed430c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -476,7 +476,8 @@ GetDatabaseCollation(Oid db_oid) } TupleDesc tupdesc = RelationGetDescr(rel); - Datum collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); + Datum collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, + &isNull); if (isNull) { info.collation = NULL; @@ -498,7 +499,8 @@ GetDatabaseCollation(Oid db_oid) #if PG_VERSION_NUM >= PG_VERSION_15 - Datum icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); + Datum icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, + &isNull); if (isNull) { info.icu_locale = NULL; @@ -508,7 +510,8 @@ GetDatabaseCollation(Oid db_oid) info.icu_locale = TextDatumGetCString(icuLocaleDatum); } - Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); + Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, + &isNull); if (isNull) { info.collversion = NULL; @@ -546,6 +549,7 @@ FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) #endif } + #if PG_VERSION_NUM >= PG_VERSION_15 static char * get_locale_provider_string(char datlocprovider) @@ -571,6 +575,8 @@ get_locale_provider_string(char datlocprovider) return ""; } } + + #endif From b28210c04fe5445807f9d5eac9b8fd9566fbc343 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 11:03:05 +0300 Subject: [PATCH 093/180] Adds shard selects to debug --- src/test/regress/sql/pg15.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 845d5fab593..109a2da414a 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1047,8 +1047,11 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +select * from pg_dist_partition; +select * from pg_dist_placement; +select * from pg_dist_shard; -select 1 from citus_remove_node('localhost', :worker_2_port); +select citus_remove_node('localhost', :worker_2_port); SET citus.log_remote_commands = true; From 3cb5cab8cd714a3ab642c8f44d644cfff777e193 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 14:01:23 +0300 Subject: [PATCH 094/180] Fixes test problem --- src/test/regress/multi_1_schedule | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 6cd6225ef42..2f664409a6c 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -33,6 +33,8 @@ test: ref_citus_local_fkeys test: alter_database_owner test: distributed_triggers test: create_single_shard_table +test: create_drop_database_propagation + # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential test: single_shard_table_udfs test: schema_based_sharding @@ -52,7 +54,6 @@ test: multi_read_from_secondaries test: grant_on_database_propagation test: alter_database_propagation -test: create_drop_database_propagation test: citus_shards From b4dec6eee8ebcfe53add2fa9b014a6888c6b128b Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 14:15:43 +0300 Subject: [PATCH 095/180] Fixes test --- src/test/regress/sql/pg15.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 109a2da414a..57ecae4102b 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -1047,10 +1047,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -select * from pg_dist_partition; -select * from pg_dist_placement; -select * from pg_dist_shard; - select citus_remove_node('localhost', :worker_2_port); From 66b433310c7f090d3af8e5b13570deafd8bf6ece Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 14:46:41 +0300 Subject: [PATCH 096/180] Moves pg15 tests to fix errors --- src/test/regress/multi_schedule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 65a27256687..56d61e29570 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -1,5 +1,6 @@ test: multi_test_helpers multi_test_helpers_superuser test: multi_cluster_management +test: pg15 test: create_role_propagation test: multi_create_fdw test: multi_test_catalog_views @@ -63,7 +64,6 @@ test: cte_inline recursive_view_local_table values sequences_with_different_type test: pg13 pg12 # run pg14 sequentially as it syncs metadata test: pg14 -test: pg15 test: pg15_jsonpath detect_conn_close test: pg16 test: drop_column_partitioned_table From 93dab802069ebe879dbac4a582b65cb2716b531c Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 15:33:55 +0300 Subject: [PATCH 097/180] Moves pg15 tests --- src/test/regress/multi_1_schedule | 1 + src/test/regress/sql/pg15.sql | 227 ----------------- .../pg15_create_drop_database_propagation.sql | 235 ++++++++++++++++++ 3 files changed, 236 insertions(+), 227 deletions(-) create mode 100644 src/test/regress/sql/pg15_create_drop_database_propagation.sql diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 2f664409a6c..766a1bc737e 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -34,6 +34,7 @@ test: alter_database_owner test: distributed_triggers test: create_single_shard_table test: create_drop_database_propagation +test: pg15_create_drop_database_propagation # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential test: single_shard_table_udfs diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql index 57ecae4102b..fe60222ddc4 100644 --- a/src/test/regress/sql/pg15.sql +++ b/src/test/regress/sql/pg15.sql @@ -976,230 +976,3 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; - - --- create/drop database for pg > 15 - - -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; - -\c - - - :master_port -create user create_drop_db_test_user; -set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = '' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'und' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - - -drop database mydatabase; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - -select citus_remove_node('localhost', :worker_2_port); - - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; - -CREATE DATABASE mydatabase2 - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - - -SET citus.log_remote_commands = true; -select 1 from citus_add_node('localhost', :worker_2_port); - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase2; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; - --- create a template database with all options set and allow connections false -CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = false - IS_TEMPLATE = true; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - - - -SET citus.log_remote_commands = true; - ---template databases could not be dropped so we need to change the template flag -SELECT result from run_command_on_all_nodes( - $$ - UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' - $$ -) ORDER BY result; - -; - -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - - -\c - - - :master_port -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; - -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; - -\c - - - :master_port -drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/pg15_create_drop_database_propagation.sql b/src/test/regress/sql/pg15_create_drop_database_propagation.sql new file mode 100644 index 00000000000..4fce5f9db23 --- /dev/null +++ b/src/test/regress/sql/pg15_create_drop_database_propagation.sql @@ -0,0 +1,235 @@ +-- +-- PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q + +-- create/drop database for pg > 15 + + +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; + +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = '' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'und' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + + +drop database mydatabase; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + +select citus_remove_node('localhost', :worker_2_port); + + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + +CREATE DATABASE mydatabase2 + WITH OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + + +SET citus.log_remote_commands = true; +select 1 from citus_add_node('localhost', :worker_2_port); + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase2; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + + + +SET citus.log_remote_commands = true; + +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + +; + +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; + +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; + +\c - - - :master_port +drop user create_drop_db_test_user; From d323ab8c51ce74de3f685dbcf000ada06707208d Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 15:44:22 +0300 Subject: [PATCH 098/180] Fixes pg15 sql --- src/test/regress/multi_schedule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 56d61e29570..65a27256687 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -1,6 +1,5 @@ test: multi_test_helpers multi_test_helpers_superuser test: multi_cluster_management -test: pg15 test: create_role_propagation test: multi_create_fdw test: multi_test_catalog_views @@ -64,6 +63,7 @@ test: cte_inline recursive_view_local_table values sequences_with_different_type test: pg13 pg12 # run pg14 sequentially as it syncs metadata test: pg14 +test: pg15 test: pg15_jsonpath detect_conn_close test: pg16 test: drop_column_partitioned_table From 736894472e653bf60d3afab919defa2346fb44de Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 16:02:09 +0300 Subject: [PATCH 099/180] Fixes pg15 tests --- src/test/regress/expected/pg15.out | 264 ---------------- .../pg15_create_drop_database_propagation.out | 284 ++++++++++++++++++ .../pg15_create_drop_database_propagation.sql | 1 + 3 files changed, 285 insertions(+), 264 deletions(-) create mode 100644 src/test/regress/expected/pg15_create_drop_database_propagation.out diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out index e58387551e8..fcbb0cd1220 100644 --- a/src/test/regress/expected/pg15.out +++ b/src/test/regress/expected/pg15.out @@ -1536,267 +1536,3 @@ SET client_min_messages TO ERROR; DROP SCHEMA pg15 CASCADE; DROP ROLE rls_tenant_1; DROP ROLE rls_tenant_2; --- create/drop database for pg > 15 -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :master_port -create user create_drop_db_test_user; -set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = '' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'und' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -drop database mydatabase; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - - - -(3 rows) - -select 1 from citus_remove_node('localhost', :worker_2_port); - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -CREATE DATABASE mydatabase2 - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] -(2 rows) - -SET citus.log_remote_commands = true; -select 1 from citus_add_node('localhost', :worker_2_port); -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';select pg_catalog.citus_internal_database_command( 'CREATE DATABASE postgres OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE regression OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template1 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template0 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''false'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE mydatabase2 OWNER = create_drop_db_test_user ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' COLLATION_VERSION = ''1.0'' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');ALTER ROLE ALL IN DATABASE regression SET lc_messages = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_monetary = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_numeric = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_time = 'C';ALTER ROLE ALL IN DATABASE regression SET bytea_output = 'hex';ALTER ROLE ALL IN DATABASE regression SET timezone_abbreviations = 'Default';SET citus.enable_ddl_propagation TO 'on' - ?column? ---------------------------------------------------------------------- - 1 -(1 row) - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase2; -NOTICE: issuing DROP DATABASE mydatabase2 -NOTICE: issuing DROP DATABASE mydatabase2 -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - - - -(3 rows) - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; --- create a template database with all options set and allow connections false -CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = false - IS_TEMPLATE = true; -NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true -NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -SET citus.log_remote_commands = true; ---template databases could not be dropped so we need to change the template flag -SELECT result from run_command_on_all_nodes( - $$ - UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - UPDATE 1 - UPDATE 1 - UPDATE 1 -(3 rows) - -; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; -NOTICE: issuing DROP DATABASE my_template_database -NOTICE: issuing DROP DATABASE my_template_database -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - - - -(3 rows) - -\c - - - :master_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; -\c - - - :master_port -drop user create_drop_db_test_user; diff --git a/src/test/regress/expected/pg15_create_drop_database_propagation.out b/src/test/regress/expected/pg15_create_drop_database_propagation.out new file mode 100644 index 00000000000..4c873142138 --- /dev/null +++ b/src/test/regress/expected/pg15_create_drop_database_propagation.out @@ -0,0 +1,284 @@ +-- +-- PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q +\endif +-- create/drop database for pg > 15 +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_1_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :worker_2_port +\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' +CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port +create user create_drop_db_test_user; +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +CREATE DATABASE mydatabase + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = '' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'und' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +drop database mydatabase; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + +(3 rows) + +select citus_remove_node('localhost', :worker_2_port); + citus_remove_node +--------------------------------------------------------------------- + +(1 row) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +CREATE DATABASE mydatabase2 + WITH OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false + OID = 966345; +NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(2 rows) + +SET citus.log_remote_commands = true; +select 1 from citus_add_node('localhost', :worker_2_port); +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';select pg_catalog.citus_internal_database_command( 'CREATE DATABASE postgres OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE regression OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template1 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template0 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''false'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE mydatabase2 OWNER = create_drop_db_test_user ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' COLLATION_VERSION = ''1.0'' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');ALTER ROLE ALL IN DATABASE regression SET lc_messages = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_monetary = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_numeric = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_time = 'C';ALTER ROLE ALL IN DATABASE regression SET bytea_output = 'hex';ALTER ROLE ALL IN DATABASE regression SET timezone_abbreviations = 'Default';SET citus.enable_ddl_propagation TO 'on' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase2' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database mydatabase2; +NOTICE: issuing DROP DATABASE mydatabase2 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE mydatabase2 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'mydatabase' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; +NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + UPDATE 1 + UPDATE 1 + UPDATE 1 +(3 rows) + +; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; +NOTICE: issuing DROP DATABASE my_template_database +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE my_template_database +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + +(3 rows) + +\c - - - :master_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_1_port +drop tablespace create_drop_db_tablespace; +\c - - - :worker_2_port +drop tablespace create_drop_db_tablespace; +\c - - - :master_port +drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/pg15_create_drop_database_propagation.sql b/src/test/regress/sql/pg15_create_drop_database_propagation.sql index 4fce5f9db23..3c7ac417ec9 100644 --- a/src/test/regress/sql/pg15_create_drop_database_propagation.sql +++ b/src/test/regress/sql/pg15_create_drop_database_propagation.sql @@ -7,6 +7,7 @@ SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 \if :server_version_ge_15 \else \q +\endif -- create/drop database for pg > 15 From a0da426a78d876e58dcbbb085b0e2ad01fa2ad37 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 16:51:31 +0300 Subject: [PATCH 100/180] Fixes tests --- .../create_drop_database_propagation.out | 109 ++++++++++++++++++ .../pg15_create_drop_database_propagation.out | 53 ++++++++- .../sql/create_drop_database_propagation.sql | 88 +++++++++++++- .../pg15_create_drop_database_propagation.sql | 37 +++++- 4 files changed, 279 insertions(+), 8 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 9f98c740c4e..add4ddac7e1 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -1,3 +1,6 @@ +-- test for create/drop database propagation +-- This test is only executes for Postgres 14 +-- For postgres 15 tests, pg15_create_drop_database_propagation.sql is used \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; \c - - - :worker_1_port @@ -154,6 +157,112 @@ SELECT result from run_command_on_all_nodes( +(3 rows) + +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; +ERROR: cannot drop a template database +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] +(3 rows) + +SET citus.log_remote_commands = true; +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + UPDATE 1 + UPDATE 1 + UPDATE 1 +(3 rows) + +; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; +NOTICE: issuing DROP DATABASE my_template_database +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE my_template_database +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + + + (3 rows) --tests for special characters in database name diff --git a/src/test/regress/expected/pg15_create_drop_database_propagation.out b/src/test/regress/expected/pg15_create_drop_database_propagation.out index 4c873142138..e1073f980b4 100644 --- a/src/test/regress/expected/pg15_create_drop_database_propagation.out +++ b/src/test/regress/expected/pg15_create_drop_database_propagation.out @@ -42,6 +42,7 @@ NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -63,7 +64,14 @@ SELECT result from run_command_on_all_nodes( [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] (3 rows) +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; drop database mydatabase; +NOTICE: issuing DROP DATABASE mydatabase +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE mydatabase +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -128,16 +136,12 @@ SELECT result from run_command_on_all_nodes( [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] (2 rows) -SET citus.log_remote_commands = true; select 1 from citus_add_node('localhost', :worker_2_port); -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';select pg_catalog.citus_internal_database_command( 'CREATE DATABASE postgres OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE regression OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template1 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE template0 OWNER = postgres ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' TABLESPACE = pg_default ALLOW_CONNECTIONS = ''false'' IS_TEMPLATE = ''true''');select pg_catalog.citus_internal_database_command( 'CREATE DATABASE mydatabase2 OWNER = create_drop_db_test_user ENCODING = ''UTF8'' LC_COLLATE = ''C'' LC_CTYPE = ''C'' LOCALE_PROVIDER = ''libc'' COLLATION_VERSION = ''1.0'' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = ''true'' IS_TEMPLATE = ''false''');ALTER ROLE ALL IN DATABASE regression SET lc_messages = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_monetary = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_numeric = 'C';ALTER ROLE ALL IN DATABASE regression SET lc_time = 'C';ALTER ROLE ALL IN DATABASE regression SET bytea_output = 'hex';ALTER ROLE ALL IN DATABASE regression SET timezone_abbreviations = 'Default';SET citus.enable_ddl_propagation TO 'on' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx ?column? --------------------------------------------------------------------- 1 (1 row) -SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -231,6 +235,32 @@ SELECT result from run_command_on_all_nodes( [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] (3 rows) +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; +ERROR: cannot drop a template database +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] +(3 rows) + SET citus.log_remote_commands = true; --template databases could not be dropped so we need to change the template flag SELECT result from run_command_on_all_nodes( @@ -274,6 +304,21 @@ SELECT result from run_command_on_all_nodes( (3 rows) +--tests for special characters in database name +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; +create database "mydatabase#1'2"; +NOTICE: issuing CREATE DATABASE "mydatabase#1'2" +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE DATABASE "mydatabase#1'2" +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database if exists "mydatabase#1'2"; +NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx \c - - - :master_port drop tablespace create_drop_db_tablespace; \c - - - :worker_1_port diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 6b95fd83ea8..d7174808ed6 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -1,5 +1,7 @@ - +-- test for create/drop database propagation +-- This test is only executes for Postgres 14 +-- For postgres 15 tests, pg15_create_drop_database_propagation.sql is used \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; @@ -135,6 +137,90 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +-- create a template database with all options set and allow connections false +CREATE DATABASE my_template_database + WITH TEMPLATE = 'template0' + OWNER = create_drop_db_test_user + ENCODING = 'UTF8' + STRATEGY = 'wal_log' + LOCALE = 'en_US.utf8' + LC_COLLATE = 'POSIX' + LC_CTYPE = 'POSIX' + ICU_LOCALE = 'en-US' + LOCALE_PROVIDER = 'icu' + COLLATION_VERSION = '1.0' + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = false + IS_TEMPLATE = true; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; + +SET citus.log_remote_commands = false; + +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; + +--template databases could not be dropped so we need to change the template flag +SELECT result from run_command_on_all_nodes( + $$ + UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' + $$ +) ORDER BY result; + +; + +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; + --tests for special characters in database name set citus.enable_create_database_propagation=on; SET citus.log_remote_commands = true; diff --git a/src/test/regress/sql/pg15_create_drop_database_propagation.sql b/src/test/regress/sql/pg15_create_drop_database_propagation.sql index 3c7ac417ec9..3a8e80ebf59 100644 --- a/src/test/regress/sql/pg15_create_drop_database_propagation.sql +++ b/src/test/regress/sql/pg15_create_drop_database_propagation.sql @@ -45,6 +45,8 @@ CREATE DATABASE mydatabase IS_TEMPLATE = false OID = 966345; +SET citus.log_remote_commands = false; + SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -60,9 +62,11 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; drop database mydatabase; +SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -114,10 +118,8 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; -SET citus.log_remote_commands = true; select 1 from citus_add_node('localhost', :worker_2_port); -SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -190,7 +192,26 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database my_template_database; + +SET citus.log_remote_commands = false; +SELECT result from run_command_on_all_nodes( + $$ + SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( + SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, + pd.datistemplate, pd.datallowconn, pd.datconnlimit, + pd.datcollate , pd. datctype , pd.datacl, + pa.rolname AS database_owner, pt.spcname AS tablespace + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + join pg_tablespace pt on pd.dattablespace = pt.oid + WHERE datname = 'my_template_database' + ) q2 + $$ +) ORDER BY result; SET citus.log_remote_commands = true; @@ -223,6 +244,16 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; +--tests for special characters in database name +set citus.enable_create_database_propagation=on; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + +create database "mydatabase#1'2"; + +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database if exists "mydatabase#1'2"; + \c - - - :master_port drop tablespace create_drop_db_tablespace; From 8519e7d7fafdfb278789a8132807d7c9c54d4242 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 16:56:25 +0300 Subject: [PATCH 101/180] Adds a dummy file for pg14 --- .../expected/pg15_create_drop_database_propagation_0.out | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/test/regress/expected/pg15_create_drop_database_propagation_0.out diff --git a/src/test/regress/expected/pg15_create_drop_database_propagation_0.out b/src/test/regress/expected/pg15_create_drop_database_propagation_0.out new file mode 100644 index 00000000000..b1ed9cc5b6c --- /dev/null +++ b/src/test/regress/expected/pg15_create_drop_database_propagation_0.out @@ -0,0 +1,9 @@ +-- +-- PG15 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 +\gset +\if :server_version_ge_15 +\else +\q From 6eeeddd5490bb65cca4d46ba00508c5ae073d498 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 17:11:01 +0300 Subject: [PATCH 102/180] Fixes pg14 tests --- .../regress/expected/create_drop_database_propagation.out | 6 +----- src/test/regress/sql/create_drop_database_propagation.sql | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index add4ddac7e1..ffd85f7c548 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -162,15 +162,11 @@ SELECT result from run_command_on_all_nodes( -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' LOCALE = 'en_US.utf8' LC_COLLATE = 'POSIX' LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index d7174808ed6..a818800925c 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -140,15 +140,11 @@ SELECT result from run_command_on_all_nodes( -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' LOCALE = 'en_US.utf8' LC_COLLATE = 'POSIX' LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; From 8c29444bbf0728e153383f202404f445453de9af Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 17:25:56 +0300 Subject: [PATCH 103/180] Fixes pg14 tests --- .../create_drop_database_propagation.out | 18 ++++++++---------- .../sql/create_drop_database_propagation.sql | 2 -- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index ffd85f7c548..239048eef38 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -165,8 +165,6 @@ CREATE DATABASE my_template_database OWNER = create_drop_db_test_user ENCODING = 'UTF8' LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; @@ -185,11 +183,11 @@ SELECT result from run_command_on_all_nodes( ) q2 $$ ) ORDER BY result; - result + result --------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] (3 rows) SET citus.log_remote_commands = true; @@ -211,11 +209,11 @@ SELECT result from run_command_on_all_nodes( ) q2 $$ ) ORDER BY result; - result + result --------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] (3 rows) SET citus.log_remote_commands = true; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index a818800925c..8e898e420f9 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -143,8 +143,6 @@ CREATE DATABASE my_template_database OWNER = create_drop_db_test_user ENCODING = 'UTF8' LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; From 2b3a556f3400f5871901f130194e6a37db8b70dd Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 17:40:37 +0300 Subject: [PATCH 104/180] Fixes pg14 tests --- .../create_drop_database_propagation.out | 40 ++++--------------- .../sql/create_drop_database_propagation.sql | 28 ++----------- 2 files changed, 10 insertions(+), 58 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 239048eef38..fd7173ce232 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -164,7 +164,8 @@ CREATE DATABASE my_template_database WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user ENCODING = 'UTF8' - LOCALE = 'en_US.utf8' + LC_COLLATE = 'C' + LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; @@ -183,40 +184,13 @@ SELECT result from run_command_on_all_nodes( ) q2 $$ ) ORDER BY result; - result + result --------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] (3 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; -ERROR: cannot drop a template database -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "en_US.utf8", "encoding": "UTF8", "datcollate": "en_US.utf8", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -SET citus.log_remote_commands = true; --template databases could not be dropped so we need to change the template flag SELECT result from run_command_on_all_nodes( $$ @@ -230,7 +204,7 @@ SELECT result from run_command_on_all_nodes( UPDATE 1 (3 rows) -; +SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database my_template_database; NOTICE: issuing DROP DATABASE my_template_database diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 8e898e420f9..540d6a9e619 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -142,7 +142,8 @@ CREATE DATABASE my_template_database WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user ENCODING = 'UTF8' - LOCALE = 'en_US.utf8' + LC_COLLATE = 'C' + LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; @@ -164,29 +165,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; - --template databases could not be dropped so we need to change the template flag SELECT result from run_command_on_all_nodes( $$ @@ -194,7 +172,7 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -; +SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database my_template_database; From cf7efca5463ad7c9c0cc87ef8b9e04ad7d94346b Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 18:01:07 +0300 Subject: [PATCH 105/180] Fixes single_node for pg14 --- src/test/regress/expected/single_node_0.out | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/single_node_0.out b/src/test/regress/expected/single_node_0.out index 321d283f85d..a44460cca78 100644 --- a/src/test/regress/expected/single_node_0.out +++ b/src/test/regress/expected/single_node_0.out @@ -88,8 +88,9 @@ SELECT create_distributed_table('failover_to_local', 'a', shard_count=>32); (1 row) CREATE INDEX CONCURRENTLY ON failover_to_local(a); -WARNING: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. - Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index. +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then reattempt the original command. ERROR: the total number of connections on the server is more than max_connections(100) HINT: Consider using a higher value for max_connections -- reset global GUC changes From 5381aa8fda9a002e51009133d75fd6cbf6f64930 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 18:13:17 +0300 Subject: [PATCH 106/180] Fixes flaky tests --- src/test/regress/citus_tests/run_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 6528834ae3d..ace48a8b99f 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -135,6 +135,7 @@ def extra_tests(self): "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "alter_role_propagation": TestDeps("minimal_schedule"), + "background_rebalance": TestDeps( None, [ @@ -151,6 +152,7 @@ def extra_tests(self): ], worker_count=6, ), + "create_drop_database_propagation": TestDeps("minimal_schedule"), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), From dcae4c3486b2137aa6b764958020e8d96e77f808 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 18:17:58 +0300 Subject: [PATCH 107/180] Fixes python code format --- src/test/regress/citus_tests/run_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index ace48a8b99f..946676e698a 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -135,7 +135,6 @@ def extra_tests(self): "minimal_schedule", ["multi_behavioral_analytics_create_table"] ), "alter_role_propagation": TestDeps("minimal_schedule"), - "background_rebalance": TestDeps( None, [ From 89e2d63bd39b0224c2f24721f908aa0c9a0e769f Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 18:37:22 +0300 Subject: [PATCH 108/180] Adds pg15 create drop to minimal tests --- src/test/regress/citus_tests/run_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 946676e698a..70844c7026a 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -152,6 +152,7 @@ def extra_tests(self): worker_count=6, ), "create_drop_database_propagation": TestDeps("minimal_schedule"), + "pg_15_create_drop_database_propagation": TestDeps("minimal_schedule"), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), From fe136886aaddc4262874530119e111c602aec619 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 18:43:51 +0300 Subject: [PATCH 109/180] Changes pg15 test files --- src/test/regress/citus_tests/run_test.py | 2 +- ...ropagation.out => create_drop_database_propagation_pg15.out} | 0 ...gation_0.out => create_drop_database_propagation_pg15_0.out} | 0 src/test/regress/multi_1_schedule | 2 +- ...ropagation.sql => create_drop_database_propagation_pg15.sql} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename src/test/regress/expected/{pg15_create_drop_database_propagation.out => create_drop_database_propagation_pg15.out} (100%) rename src/test/regress/expected/{pg15_create_drop_database_propagation_0.out => create_drop_database_propagation_pg15_0.out} (100%) rename src/test/regress/sql/{pg15_create_drop_database_propagation.sql => create_drop_database_propagation_pg15.sql} (100%) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 70844c7026a..30f6b3b379b 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -152,7 +152,7 @@ def extra_tests(self): worker_count=6, ), "create_drop_database_propagation": TestDeps("minimal_schedule"), - "pg_15_create_drop_database_propagation": TestDeps("minimal_schedule"), + "create_drop_database_propagation_pg_15": TestDeps("minimal_schedule"), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), diff --git a/src/test/regress/expected/pg15_create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation_pg15.out similarity index 100% rename from src/test/regress/expected/pg15_create_drop_database_propagation.out rename to src/test/regress/expected/create_drop_database_propagation_pg15.out diff --git a/src/test/regress/expected/pg15_create_drop_database_propagation_0.out b/src/test/regress/expected/create_drop_database_propagation_pg15_0.out similarity index 100% rename from src/test/regress/expected/pg15_create_drop_database_propagation_0.out rename to src/test/regress/expected/create_drop_database_propagation_pg15_0.out diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 766a1bc737e..3ad0eabfc99 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -34,7 +34,7 @@ test: alter_database_owner test: distributed_triggers test: create_single_shard_table test: create_drop_database_propagation -test: pg15_create_drop_database_propagation +test: create_drop_database_propagation_pg15 # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential test: single_shard_table_udfs diff --git a/src/test/regress/sql/pg15_create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation_pg15.sql similarity index 100% rename from src/test/regress/sql/pg15_create_drop_database_propagation.sql rename to src/test/regress/sql/create_drop_database_propagation_pg15.sql From 693eeedfaf0f7b8189174a9bf95b96532b56ed59 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 27 Oct 2023 19:08:09 +0300 Subject: [PATCH 110/180] Fixes pg15 test name --- src/test/regress/citus_tests/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 30f6b3b379b..90c16b04e21 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -152,7 +152,7 @@ def extra_tests(self): worker_count=6, ), "create_drop_database_propagation": TestDeps("minimal_schedule"), - "create_drop_database_propagation_pg_15": TestDeps("minimal_schedule"), + "create_drop_database_propagation_pg15": TestDeps("minimal_schedule"), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), From d8639d58de527f9a14abf768ab945c9fe2e07b46 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 31 Oct 2023 10:58:44 +0300 Subject: [PATCH 111/180] Adds locale restrictions --- src/backend/distributed/commands/database.c | 17 +++++++ .../commands/distribute_object_ops.c | 2 +- .../deparser/deparse_database_stmts.c | 15 +++++++ src/include/distributed/commands.h | 2 + .../create_drop_database_propagation.out | 40 ++++++++++++----- .../create_drop_database_propagation_pg15.out | 38 +++------------- .../sql/create_drop_database_propagation.sql | 44 ++++++++++++++----- .../create_drop_database_propagation_pg15.sql | 27 +----------- 8 files changed, 106 insertions(+), 79 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 42ef2ed430c..4867c55c233 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -263,6 +263,23 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } +List * +PreprocessCreateDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext) +{ + if (!EnableCreateDatabasePropagation || !ShouldPropagate()) + { + return NIL; + } + + EnsureCoordinator(); + + //Validate the statement + DeparseTreeNode(node); + + return NIL; +} + /* * PostprocessCreatedbStmt is executed after the statement is applied to the local diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index 2888bef1d13..37ad4c5a45d 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -469,7 +469,7 @@ static DistributeObjectOps Database_Alter = { static DistributeObjectOps Database_Create = { .deparse = DeparseCreateDatabaseStmt, .qualify = NULL, - .preprocess = NULL, + .preprocess = PreprocessCreateDatabaseStmt, .postprocess = PostprocessCreateDatabaseStmt, .objectType = OBJECT_DATABASE, .operationType = DIST_OPS_CREATE, diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index bf98b962251..169ca40e893 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -237,6 +237,21 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { + //If option is template, lc_type, locale or lc_collate, propagation will not be supportted + // since template database is not stored in the catalog + if (strcmp(option->defname, "template") == 0 || + strcmp(option->defname, "strategy") == 0 || + strcmp(option->defname, "lc_ctype") == 0 || + strcmp(option->defname, "locale") == 0 || + strcmp(option->defname, "lc_collate") == 0 || + strcmp(option->defname, "icu_locale") == 0 || + strcmp(option->defname, "locale_provider") == 0 ) + { + ereport(ERROR, + errmsg("CREATE DATABASE option \"%s\" is not supported", + option->defname)); + } + optionToStatement(buf, option, create_database_option_formats, lengthof( create_database_option_formats)); } diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index a4f890f9ea3..28828075b5c 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -237,6 +237,8 @@ extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); +extern List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, + ProcessUtilityContext processUtilityContext); extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index fd7173ce232..9489664eb42 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -14,11 +14,8 @@ create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; CREATE DATABASE mydatabase WITH OWNER = create_drop_db_test_user - TEMPLATE = 'template0' ENCODING = 'UTF8' CONNECTION LIMIT = 10 - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; @@ -74,12 +71,9 @@ select 1 from citus_remove_node('localhost', :worker_2_port); --test with is_template true and allow connections false CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = false; @@ -161,11 +155,8 @@ SELECT result from run_command_on_all_nodes( -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; @@ -248,6 +239,35 @@ NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +--test for unsupported options +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_CTYPE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; +ERROR: CREATE DATABASE option "lc_ctype" is not supported +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_CTYPE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; +ERROR: CREATE DATABASE option "lc_ctype" is not supported +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_COLLATE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; +ERROR: CREATE DATABASE option "lc_collate" is not supported +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LOCALE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; +ERROR: CREATE DATABASE option "locale" is not supported --clean up resources created by this test drop tablespace create_drop_db_tablespace; \c - - - :worker_1_port diff --git a/src/test/regress/expected/create_drop_database_propagation_pg15.out b/src/test/regress/expected/create_drop_database_propagation_pg15.out index e1073f980b4..bc637480350 100644 --- a/src/test/regress/expected/create_drop_database_propagation_pg15.out +++ b/src/test/regress/expected/create_drop_database_propagation_pg15.out @@ -23,24 +23,17 @@ set citus.enable_create_database_propagation=on; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' + WITH OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = '' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'und' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +NOTICE: issuing CREATE DATABASE mydatabase OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE DATABASE mydatabase TEMPLATE template0 OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE '' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'und' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +NOTICE: issuing CREATE DATABASE mydatabase OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( @@ -104,16 +97,11 @@ set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase2 WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 +NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( @@ -196,22 +184,15 @@ SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE DATABASE%'; -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; -NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +NOTICE: issuing CREATE DATABASE my_template_database OWNER create_drop_db_test_user ENCODING 'UTF8' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE DATABASE my_template_database TEMPLATE template0 OWNER create_drop_db_test_user ENCODING 'UTF8' STRATEGY 'wal_log' LOCALE 'en_US.utf8' LC_COLLATE 'POSIX' LC_CTYPE 'POSIX' ICU_LOCALE 'en-US' LOCALE_PROVIDER 'icu' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true +NOTICE: issuing CREATE DATABASE my_template_database OWNER create_drop_db_test_user ENCODING 'UTF8' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( @@ -235,10 +216,6 @@ SELECT result from run_command_on_all_nodes( [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] (3 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; -ERROR: cannot drop a template database SET citus.log_remote_commands = false; SELECT result from run_command_on_all_nodes( $$ @@ -275,7 +252,6 @@ SELECT result from run_command_on_all_nodes( UPDATE 1 (3 rows) -; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database my_template_database; NOTICE: issuing DROP DATABASE my_template_database diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 540d6a9e619..ae90088d1b6 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -21,17 +21,12 @@ set citus.enable_create_database_propagation=on; CREATE DATABASE mydatabase WITH OWNER = create_drop_db_test_user - TEMPLATE = 'template0' ENCODING = 'UTF8' CONNECTION LIMIT = 10 - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; - - SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -73,12 +68,9 @@ select 1 from citus_remove_node('localhost', :worker_2_port); --test with is_template true and allow connections false CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = false; @@ -139,11 +131,8 @@ SELECT result from run_command_on_all_nodes( -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - LC_COLLATE = 'C' - LC_CTYPE = 'C' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false IS_TEMPLATE = true; @@ -203,6 +192,37 @@ create database "mydatabase#1'2"; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database if exists "mydatabase#1'2"; +--test for unsupported options + +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_CTYPE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; + +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_CTYPE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; + +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LC_COLLATE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; + +CREATE DATABASE mydatabase + with CONNECTION LIMIT = 10 + ENCODING = 'UTF8' + LOCALE = 'C.UTF-8' + ALLOW_CONNECTIONS = false + IS_TEMPLATE = false; + + --clean up resources created by this test drop tablespace create_drop_db_tablespace; diff --git a/src/test/regress/sql/create_drop_database_propagation_pg15.sql b/src/test/regress/sql/create_drop_database_propagation_pg15.sql index 3a8e80ebf59..ca3e3b20235 100644 --- a/src/test/regress/sql/create_drop_database_propagation_pg15.sql +++ b/src/test/regress/sql/create_drop_database_propagation_pg15.sql @@ -29,17 +29,10 @@ set citus.enable_create_database_propagation=on; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase - WITH TEMPLATE = 'template0' + WITH OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = '' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'und' - LOCALE_PROVIDER = 'icu' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false @@ -91,11 +84,6 @@ set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase2 WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false @@ -161,15 +149,8 @@ set citus.grep_remote_commands = '%CREATE DATABASE%'; -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database - WITH TEMPLATE = 'template0' - OWNER = create_drop_db_test_user + WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - STRATEGY = 'wal_log' - LOCALE = 'en_US.utf8' - LC_COLLATE = 'POSIX' - LC_CTYPE = 'POSIX' - ICU_LOCALE = 'en-US' - LOCALE_PROVIDER = 'icu' COLLATION_VERSION = '1.0' TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = false @@ -192,9 +173,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; SET citus.log_remote_commands = false; @@ -222,7 +200,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database my_template_database; From 2d9181a9ab332a34d117712fcf199401fb3619e0 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 31 Oct 2023 11:00:40 +0300 Subject: [PATCH 112/180] Fixes indents --- src/backend/distributed/commands/database.c | 5 +++-- src/backend/distributed/deparser/deparse_database_stmts.c | 6 +++--- src/include/distributed/commands.h | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 4867c55c233..22de2d4bd52 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -263,9 +263,10 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, return NodeDDLTaskList(NON_COORDINATOR_NODES, commands); } + List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext) + ProcessUtilityContext processUtilityContext) { if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { @@ -274,7 +275,7 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - //Validate the statement + /*Validate the statement */ DeparseTreeNode(node); return NIL; diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 169ca40e893..e286dae650a 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -237,15 +237,15 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - //If option is template, lc_type, locale or lc_collate, propagation will not be supportted - // since template database is not stored in the catalog + /*If option is template, lc_type, locale or lc_collate, propagation will not be supportted */ + /* since template database is not stored in the catalog */ if (strcmp(option->defname, "template") == 0 || strcmp(option->defname, "strategy") == 0 || strcmp(option->defname, "lc_ctype") == 0 || strcmp(option->defname, "locale") == 0 || strcmp(option->defname, "lc_collate") == 0 || strcmp(option->defname, "icu_locale") == 0 || - strcmp(option->defname, "locale_provider") == 0 ) + strcmp(option->defname, "locale_provider") == 0) { ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 28828075b5c..0e43fa386cb 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -238,7 +238,7 @@ extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString ProcessUtilityContext processUtilityContext); extern List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); + ProcessUtilityContext processUtilityContext); extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); From c72f5357e1bcdd92a77d1fa9eb7aea521a0c881f Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Tue, 31 Oct 2023 16:10:03 +0100 Subject: [PATCH 113/180] Fix PG16 assertion --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 22de2d4bd52..ceac576afb1 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -55,7 +55,7 @@ List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); /* controlled via GUC */ -bool EnableCreateDatabasePropagation = true; +bool EnableCreateDatabasePropagation = false; bool EnableAlterDatabaseOwner = true; /* From 65660db10dc3a587db6bfc1bb6273302b521ffc8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 8 Nov 2023 02:02:00 +0300 Subject: [PATCH 114/180] Fixes review items --- src/backend/distributed/commands/common.c | 4 +- src/backend/distributed/commands/database.c | 199 ++++++++---------- src/backend/distributed/commands/role.c | 1 + .../distributed/commands/utility_hook.c | 32 +-- .../distributed/deparser/citus_deparseutils.c | 48 +++-- .../deparser/deparse_database_stmts.c | 93 ++++++-- src/backend/distributed/metadata/distobject.c | 15 +- .../distributed/metadata/metadata_sync.c | 80 ++++++- src/backend/distributed/shared_library_init.c | 2 +- .../12.2-1.sql | 2 +- .../latest.sql | 2 +- src/include/distributed/commands.h | 12 +- .../distributed/commands/utility_hook.h | 2 +- src/include/distributed/deparser.h | 16 +- src/include/distributed/metadata/distobject.h | 3 +- src/test/regress/citus_tests/run_test.py | 2 - 16 files changed, 309 insertions(+), 204 deletions(-) diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index 9a87df9f103..957e2616128 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -14,8 +14,10 @@ #include "postgres.h" #include "catalog/objectaddress.h" +#include "catalog/pg_database.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" +#include "commands/dbcommands.h" #include "nodes/parsenodes.h" #include "tcop/utility.h" @@ -28,8 +30,6 @@ #include "distributed/metadata/distobject.h" #include "distributed/multi_executor.h" #include "distributed/worker_transaction.h" -#include "catalog/pg_database.h" -#include "commands/dbcommands.h" /* diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index ceac576afb1..cabe69fd6dc 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -10,46 +10,53 @@ */ #include "postgres.h" +#include "miscadmin.h" +#include "access/heapam.h" #include "access/htup_details.h" #include "access/xact.h" #include "catalog/objectaddress.h" +#include "catalog/pg_collation.h" #include "catalog/pg_database.h" +#include "catalog/pg_database_d.h" +#include "catalog/pg_tablespace.h" #include "commands/dbcommands.h" -#include "miscadmin.h" #include "nodes/parsenodes.h" -#include "utils/syscache.h" #include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/relcache.h" +#include "utils/syscache.h" +#include "distributed/adaptive_executor.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" +#include "distributed/deparse_shard_query.h" #include "distributed/deparser.h" +#include "distributed/listutils.h" +#include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" #include "distributed/metadata_utility.h" #include "distributed/multi_executor.h" #include "distributed/relation_access_tracking.h" -#include "distributed/worker_transaction.h" -#include "distributed/deparser.h" #include "distributed/worker_protocol.h" -#include "distributed/metadata/distobject.h" -#include "distributed/deparse_shard_query.h" -#include "distributed/listutils.h" -#include "distributed/adaptive_executor.h" -#include "access/htup_details.h" -#include "catalog/pg_tablespace.h" -#include "access/heapam.h" -#include "utils/relcache.h" -#include "utils/rel.h" -#include "utils/lsyscache.h" -#include "catalog/pg_collation.h" -#include "utils/relcache.h" -#include "catalog/pg_database_d.h" - +#include "distributed/worker_transaction.h" -static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); +/* + * DatabaseCollationInfo is used to store collation related information of a database + */ +typedef struct DatabaseCollationInfo +{ + char *collation; + char *ctype; + #if PG_VERSION_NUM >= PG_VERSION_15 + char *icu_locale; + char *collversion; + #endif +} DatabaseCollationInfo; -PG_FUNCTION_INFO_V1(citus_internal_database_command); +static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); static Oid get_database_owner(Oid db_oid); List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); @@ -264,6 +271,13 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, } +/* + * PostprocessAlterDatabaseStmt is executed before the statement is applied to the local + * postgres instance. + * + * In this stage, we can perform validations and prepare the commands that need to + * be run on all workers to grant. + */ List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) @@ -304,82 +318,21 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) (void *) createDatabaseCommand, ENABLE_DDL_PROPAGATION); - return NontransactionalNodeDDLTask(NON_COORDINATOR_NODES, commands); + return NontransactionalNodeDDLTaskList(NON_COORDINATOR_NODES, commands); } /* - * citus_internal_database_command is an internal UDF to - * create/drop a database in an idempotent maner without - * transaction block restrictions. + * PostprocessAlterDatabaseStmt is executed after the statement is applied to the local + * postgres instance. In this stage we can prepare the commands that need to be run on + * all workers to drop the database. Since the DROP DATABASE statement gives error in + * transaction context, we need to use NontransactionalNodeDDLTaskList to send the + * DROP DATABASE statement to the workers. */ -Datum -citus_internal_database_command(PG_FUNCTION_ARGS) -{ - int saveNestLevel = NewGUCNestLevel(); - text *commandText = PG_GETARG_TEXT_P(0); - char *command = text_to_cstring(commandText); - Node *parseTree = ParseTreeNode(command); - - set_config_option("citus.enable_ddl_propagation", "off", - (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, - GUC_ACTION_LOCAL, true, 0, false); - - set_config_option("citus.enable_create_database_propagation", "off", - (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, - GUC_ACTION_LOCAL, true, 0, false); - - /* - * createdb() / DropDatabase() uses ParseState to report the error position for the - * input command and the position is reported to be 0 when it's provided as NULL. - * We're okay with that because we don't expect this UDF to be called with an incorrect - * DDL command. - * - */ - ParseState *pstate = NULL; - - if (IsA(parseTree, CreatedbStmt)) - { - CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); - - bool missingOk = true; - Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - if (!OidIsValid(databaseOid)) - { - createdb(pstate, (CreatedbStmt *) parseTree); - } - } - else if (IsA(parseTree, DropdbStmt)) - { - DropdbStmt *stmt = castNode(DropdbStmt, parseTree); - - bool missingOk = false; - Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - - if (OidIsValid(databaseOid)) - { - DropDatabase(pstate, (DropdbStmt *) parseTree); - } - } - else - { - ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); - } - - /* Below command rollbacks flags to the state before this session*/ - AtEOXact_GUC(true, saveNestLevel); - - PG_RETURN_VOID(); -} - - List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { - bool isPostProcess = false; if (!EnableCreateDatabasePropagation || !ShouldPropagate()) { return NIL; @@ -389,41 +342,50 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, DropdbStmt *stmt = (DropdbStmt *) node; + bool isPostProcess = false; List *addresses = GetObjectAddressListFromParseTree(node, stmt->missing_ok, isPostProcess); - if (list_length(addresses) == 0) + if (list_length(addresses) != 1) { - return NIL; + ereport(ERROR, (errmsg("unexpected number of objects found when " + "executing DROP DATABASE command"))); } ObjectAddress *address = (ObjectAddress *) linitial(addresses); - if (address->objectId == InvalidOid || !IsObjectDistributed(address)) + if (address->objectId == InvalidOid || !IsAnyObjectDistributed(list_make1(address))) { return NIL; } char *dropDatabaseCommand = DeparseTreeNode(node); - List *commands = list_make3(DISABLE_DDL_PROPAGATION, (void *) dropDatabaseCommand, ENABLE_DDL_PROPAGATION); - return NontransactionalNodeDDLTask(NON_COORDINATOR_NODES, commands); + return NontransactionalNodeDDLTaskList(NON_COORDINATOR_NODES, commands); } +/* + * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress + * of the database. + */ static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) { Oid databaseOid = get_database_oid(databaseName, missingOk); - ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); - ObjectAddressSet(*dbAddress, DatabaseRelationId, databaseOid); - return dbAddress; + ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, databaseOid); + return dbObjectAddress; } +/* + * DropDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the + * object of the DropdbStmt. + */ List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { @@ -434,6 +396,10 @@ DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) } +/* + * CreateDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the + * object of the CreatedbStmt. + */ List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) { @@ -444,6 +410,9 @@ CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) } +/* + * GetTablespaceName gets the tablespace oid and returns the tablespace name. + */ static char * GetTablespaceName(Oid tablespaceOid) { @@ -462,19 +431,6 @@ GetTablespaceName(Oid tablespaceOid) } -/* - * DatabaseCollationInfo is used to store collation related information of a database - */ -typedef struct DatabaseCollationInfo -{ - char *collation; - char *ctype; - #if PG_VERSION_NUM >= PG_VERSION_15 - char *icu_locale; - char *collversion; - #endif -} DatabaseCollationInfo; - /* * GetDatabaseCollation gets oid of a database and returns all the collation related information * We need this method since collation related info in Form_pg_database is not accessible @@ -548,6 +504,9 @@ GetDatabaseCollation(Oid db_oid) } +/* + * FreeDatabaseCollationInfo frees the memory allocated for DatabaseCollationInfo + */ static void FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) { @@ -569,8 +528,13 @@ FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) #if PG_VERSION_NUM >= PG_VERSION_15 + +/* + * GetLocaleProviderString gets the datlocprovider stored in pg_database + * and returns the string representation of the datlocprovider + */ static char * -get_locale_provider_string(char datlocprovider) +GetLocaleProviderString(char datlocprovider) { switch (datlocprovider) { @@ -599,7 +563,8 @@ get_locale_provider_string(char datlocprovider) /* - * GenerateCreateDatabaseStatementFromPgDatabase is gets the pg_database tuple and returns the CREATE DATABASE statement + * GenerateCreateDatabaseStatementFromPgDatabase gets the pg_database tuple and returns the + * CREATE DATABASE statement that can be used to create given database. */ static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) @@ -642,7 +607,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) if (databaseForm->datlocprovider != 0) { - appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", get_locale_provider_string( + appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", GetLocaleProviderString( databaseForm->datlocprovider)); } @@ -678,19 +643,21 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) /* - * GenerateCreateDatabaseCommandList is gets the pg_database tuples and returns the CREATE DATABASE statement list - * for all the databases in the cluster.citus_internal_database_command UDF is used to send the CREATE DATABASE - * statement to the workers since the CREATE DATABASE statement gives error in transaction context. + * GenerateCreateDatabaseCommandList gets a list of pg_database tuples and returns + * a list of CREATE DATABASE statements for all the databases. + * + * Commands in the list are wrapped by citus_internal_database_command() UDF + * to avoid from transaction block restrictions that apply to database commands */ List * GenerateCreateDatabaseCommandList(void) { List *commands = NIL; - HeapTuple tuple; Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); + HeapTuple tuple = NULL; while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); @@ -702,7 +669,7 @@ GenerateCreateDatabaseCommandList(void) /* Generate the CREATE DATABASE statement */ appendStringInfo(outerDbStmt, - "select pg_catalog.citus_internal_database_command( %s)", + "SELECT pg_catalog.citus_internal_database_command( %s)", quote_literal_cstr( createStmt)); diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 792efd93494..976cacf8954 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -65,6 +65,7 @@ static DefElem * makeDefElemBool(char *name, bool value); static List * GenerateRoleOptionsList(HeapTuple tuple); static List * GenerateGrantRoleStmtsFromOptions(RoleSpec *roleSpec, List *options); static List * GenerateGrantRoleStmtsOfRole(Oid roleid); +static void EnsureSequentialModeForRoleDDL(void); static char * GetRoleNameFromDbRoleSetting(HeapTuple tuple, TupleDesc DbRoleSettingDescription); diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 0d400d1390c..150d136761c 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -25,7 +25,6 @@ *------------------------------------------------------------------------- */ -#include "distributed/pg_version_constants.h" #include "postgres.h" #include "miscadmin.h" @@ -35,6 +34,7 @@ #include "access/htup_details.h" #include "catalog/catalog.h" #include "catalog/dependency.h" +#include "catalog/pg_database.h" #include "citus_version.h" #include "commands/dbcommands.h" #include "commands/defrem.h" @@ -62,6 +62,7 @@ #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_physical_planner.h" +#include "distributed/pg_version_constants.h" #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" #include "distributed/string_utils.h" @@ -80,7 +81,6 @@ #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/syscache.h" -#include "catalog/pg_database.h" bool EnableDDLPropagation = true; /* ddl propagation is enabled */ @@ -579,7 +579,6 @@ citus_ProcessUtilityInternal(PlannedStmt *pstmt, PreprocessLockStatement((LockStmt *) parsetree, context); } - /* * We only process ALTER TABLE ... ATTACH PARTITION commands in the function below * and distribute the partition if necessary. @@ -726,12 +725,13 @@ citus_ProcessUtilityInternal(PlannedStmt *pstmt, } /* - * Make sure that dropping the role and database deletes the pg_dist_object entries. There is a - * separate logic for roles and database, since roles and database are not included as dropped objects in the - * drop event trigger. To handle it both on worker and coordinator nodes, it is not - * implemented as a part of process functions but here. + * Make sure that dropping node-wide objects deletes the pg_dist_object + * entries. There is a separate logic for node-wide objects (such as role + * and databases), since they are not included as dropped objects in the + * drop event trigger. To handle it both on worker and coordinator nodes, + * it is not implemented as a part of process functions but here. */ - UnmarkRolesAndDatabaseDistributed(parsetree); + UnmarkNodeWideObjectsDistributed(parsetree); pstmt->utilityStmt = parsetree; @@ -1265,10 +1265,12 @@ ExecuteDistributedDDLJob(DDLJob *ddlJob) { ereport(WARNING, (errmsg( - "Commands that are not transaction-safe may result in partial failure" - ", potentially leading to an inconsistent state.\nIf the problematic command" - " is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the " - "object,\nif applicable, and then reattempt the original command."))); + "Commands that are not transaction-safe may result in " + "partial failure, potentially leading to an inconsistent " + "state.\nIf the problematic command is a CREATE operation, " + "consider using the 'IF EXISTS' syntax to drop the object," + "\nif applicable, and then re-attempt the original command."))); + PG_RE_THROW(); } } @@ -1483,12 +1485,12 @@ DDLTaskList(Oid relationId, const char *commandString) /* - * NontransactionalNodeDDLTask builds a list of tasks to execute a DDL command on a + * NontransactionalNodeDDLTaskList builds a list of tasks to execute a DDL command on a * given target set of nodes with cannotBeExecutedInTransaction is set to make sure - * that list is being executed without a transaction. + * that task list is executed outside a transaction block. */ List * -NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands) +NontransactionalNodeDDLTaskList(TargetWorkerSet targets, List *commands) { List *ddlJobs = NodeDDLTaskList(targets, commands); DDLJob *ddlJob = NULL; diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 6492c14f2ce..0cfd7dd6fcf 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -1,56 +1,66 @@ - +/* + * citus_deparseutils.c + * --------------------- + * + * This file contains common functions used for deparsing PostgreSQL statements + * to their equivalent SQL representation. + * + */ #include "postgres.h" -#include "utils/builtins.h" + #include "commands/defrem.h" +#include "distributed/deparser.h" +#include "distributed/pg_version_constants.h" +#include "utils/builtins.h" #include "utils/elog.h" #include "utils/rel.h" #include "utils/relcache.h" #include "utils/syscache.h" #include "utils/typcache.h" -#include "distributed/deparser.h" -#include "distributed/pg_version_constants.h" /** - * Convert a DefElem option to a SQL statement and append it to the given StringInfo buffer. + * DefElemOptionToStatement converts a DefElem option to a SQL statement and + * appends it to the given StringInfo buffer. * * @param buf The StringInfo buffer to append the SQL statement to. * @param option The DefElem option to convert to a SQL statement. - * @param opt_formats The option format specification to use for the conversion. - * @param num_opt_formats The number of option formats in the opt_formats array. + * @param optionFormats The option format specification to use for the conversion. + * @param optionFormatsLen The number of option formats in the opt_formats array. */ void -optionToStatement(StringInfo buf, DefElem *option, const struct - option_format *opt_formats, int - opt_formats_len) +DefElemOptionToStatement(StringInfo buf, DefElem *option, const + DefElemOptionFormat *optionFormats, int + optionFormatsLen) { const char *name = option->defname; int i; - for (i = 0; i < opt_formats_len; i++) + for (i = 0; i < optionFormatsLen; i++) { - if (strcmp(name, opt_formats[i].name) == 0) + if (strcmp(name, optionFormats[i].name) == 0) { - switch (opt_formats[i].type) + switch (optionFormats[i].type) { case OPTION_FORMAT_STRING: { char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_identifier(value)); + appendStringInfo(buf, optionFormats[i].format, quote_identifier( + value)); break; } case OPTION_FORMAT_INTEGER: { int32 value = defGetInt32(option); - appendStringInfo(buf, opt_formats[i].format, value); + appendStringInfo(buf, optionFormats[i].format, value); break; } case OPTION_FORMAT_BOOLEAN: { bool value = defGetBoolean(option); - appendStringInfo(buf, opt_formats[i].format, value ? "true" : + appendStringInfo(buf, optionFormats[i].format, value ? "true" : "false"); break; } @@ -59,7 +69,7 @@ optionToStatement(StringInfo buf, DefElem *option, const struct case OPTION_FORMAT_OBJECT_ID: { Oid value = defGetObjectId(option); - appendStringInfo(buf, opt_formats[i].format, value); + appendStringInfo(buf, optionFormats[i].format, value); break; } @@ -67,14 +77,14 @@ optionToStatement(StringInfo buf, DefElem *option, const struct case OPTION_FORMAT_LITERAL_CSTR: { char *value = defGetString(option); - appendStringInfo(buf, opt_formats[i].format, quote_literal_cstr( + appendStringInfo(buf, optionFormats[i].format, quote_literal_cstr( value)); break; } default: { - elog(ERROR, "unrecognized option type: %d", opt_formats[i].type); + elog(ERROR, "unrecognized option type: %d", optionFormats[i].type); break; } } diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index e286dae650a..e96015fdc1d 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -12,25 +12,25 @@ #include "postgres.h" #include "pg_version_compat.h" - #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" +#include "commands/defrem.h" #include "distributed/deparser.h" #include "distributed/citus_ruleutils.h" -#include "commands/defrem.h" #include "distributed/deparser.h" +#include "distributed/listutils.h" #include "distributed/log_utils.h" #include "parser/parse_type.h" -#include "distributed/listutils.h" + static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendDefElemConnLimit(StringInfo buf, DefElem *def); -const struct option_format create_database_option_formats[] = { +const DefElemOptionFormat create_database_option_formats[] = { { "owner", " OWNER %s", OPTION_FORMAT_STRING }, { "template", " TEMPLATE %s", OPTION_FORMAT_STRING }, { "encoding", " ENCODING %s", OPTION_FORMAT_LITERAL_CSTR }, @@ -49,6 +49,22 @@ const struct option_format create_database_option_formats[] = { { "oid", " OID %d", OPTION_FORMAT_OBJECT_ID } }; +/* + * DeparseAlterDatabaseOwnerStmt + * Deparse an AlterDatabaseOwnerStmt node + * + * This function is responsible for producing a string representation of an + * AlterDatabaseOwnerStmt node, which represents an ALTER DATABASE statement + * that changes the owner of a database. The output string includes the ALTER + * DATABASE keyword, the name of the database being altered, and the new owner + * of the database. + * + * Parameters: + * - node: a pointer to the AlterDatabaseOwnerStmt node to be deparsed + * + * Returns: + * - a string representation of the ALTER DATABASE statement + */ char * DeparseAlterDatabaseOwnerStmt(Node *node) { @@ -64,6 +80,15 @@ DeparseAlterDatabaseOwnerStmt(Node *node) } +/* + * + * AppendAlterDatabaseOwnerStmt + * Append an ALTER DATABASE statement for changing the owner of a database to the given StringInfo buffer. + * + * Parameters: + * - buf: The StringInfo buffer to append the statement to. + * - stmt: The AlterOwnerStmt representing the ALTER DATABASE statement to append. + */ static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { @@ -226,6 +251,32 @@ DeparseAlterDatabaseSetStmt(Node *node) } +/* + * Validates for if option is template, lc_type, locale or lc_collate, propagation will + * not be supported since template and strategy options are not stored in the catalog + * and lc_type, locale and lc_collate options depends on template parameter. + */ +static void +ValidateCreateDatabaseOptions(DefElem *option) +{ + if (strcmp(option->defname, "template") == 0 || + strcmp(option->defname, "strategy") == 0 || + strcmp(option->defname, "lc_ctype") == 0 || + strcmp(option->defname, "locale") == 0 || + strcmp(option->defname, "lc_collate") == 0 || + strcmp(option->defname, "icu_locale") == 0 || + strcmp(option->defname, "locale_provider") == 0) + { + ereport(ERROR, + errmsg("CREATE DATABASE option \"%s\" is not supported", + option->defname)); + } +} + + +/* + * Prepares a CREATE DATABASE statement with given empty StringInfo buffer and CreatedbStmt node. + */ static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -237,27 +288,18 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - /*If option is template, lc_type, locale or lc_collate, propagation will not be supportted */ - /* since template database is not stored in the catalog */ - if (strcmp(option->defname, "template") == 0 || - strcmp(option->defname, "strategy") == 0 || - strcmp(option->defname, "lc_ctype") == 0 || - strcmp(option->defname, "locale") == 0 || - strcmp(option->defname, "lc_collate") == 0 || - strcmp(option->defname, "icu_locale") == 0 || - strcmp(option->defname, "locale_provider") == 0) - { - ereport(ERROR, - errmsg("CREATE DATABASE option \"%s\" is not supported", - option->defname)); - } + ValidateCreateDatabaseOptions(option); - optionToStatement(buf, option, create_database_option_formats, lengthof( - create_database_option_formats)); + DefElemOptionToStatement(buf, option, create_database_option_formats, + lengthof(create_database_option_formats)); } } +/* + * Converts a CreatedbStmt structure into a SQL command string. + * Used in the deparsing of Create database statement. + */ char * DeparseCreateDatabaseStmt(Node *node) { @@ -271,13 +313,16 @@ DeparseCreateDatabaseStmt(Node *node) } +/* + * Prepares a DROP DATABASE statement with given empty StringInfo buffer and DropdbStmt node. + */ static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { - char *if_exists_statement = stmt->missing_ok ? "IF EXISTS" : ""; + char *ifExistsStatement = stmt->missing_ok ? "IF EXISTS" : ""; appendStringInfo(buf, "DROP DATABASE %s %s", - if_exists_statement, + ifExistsStatement, quote_identifier(stmt->dbname)); DefElem *option = NULL; @@ -298,6 +343,10 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) } +/* + * Converts a DropdbStmt structure into a SQL command string. + * Used in the deparsing of drop database statement. + */ char * DeparseDropDatabaseStmt(Node *node) { diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index af8354ee3b8..1a0b90f2c71 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -55,6 +55,7 @@ static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, Datum *paramValues); +static bool IsObjectDistributed(const ObjectAddress *address); PG_FUNCTION_INFO_V1(citus_unmark_object_distributed); PG_FUNCTION_INFO_V1(master_unmark_object_distributed); @@ -358,8 +359,12 @@ ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, } +/* + * Deletes all pg_dist_object records for distributed roles in `DROP ROLE` statement a + * and for all databases in `DROP DATABASE` statement + */ void -UnmarkRolesAndDatabaseDistributed(Node *node) +UnmarkNodeWideObjectsDistributed(Node *node) { if (IsA(node, DropRoleStmt)) { @@ -378,9 +383,9 @@ UnmarkRolesAndDatabaseDistributed(Node *node) char *dbName = stmt->dbname; Oid dbOid = get_database_oid(dbName, stmt->missing_ok); - ObjectAddress *dbAddress = palloc0(sizeof(ObjectAddress)); - ObjectAddressSet(*dbAddress, DatabaseRelationId, dbOid); - UnmarkObjectDistributed(dbAddress); + ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, dbOid); + UnmarkObjectDistributed(dbObjectAddress); } } @@ -420,7 +425,7 @@ UnmarkObjectDistributed(const ObjectAddress *address) * IsObjectDistributed returns if the object addressed is already distributed in the * cluster. This performs a local indexed lookup in pg_dist_object. */ -bool +static bool IsObjectDistributed(const ObjectAddress *address) { ScanKeyData key[3]; diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 54fa801ae89..bc679750483 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -30,12 +30,15 @@ #include "catalog/pg_attrdef.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" +#include "catalog/pg_database.h" +#include "catalog/pg_database_d.h" #include "catalog/pg_depend.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "commands/async.h" +#include "commands/dbcommands.h" #include "distributed/argutils.h" #include "distributed/backend_data.h" #include "distributed/citus_ruleutils.h" @@ -179,6 +182,7 @@ PG_FUNCTION_INFO_V1(citus_internal_delete_colocation_metadata); PG_FUNCTION_INFO_V1(citus_internal_add_tenant_schema); PG_FUNCTION_INFO_V1(citus_internal_delete_tenant_schema); PG_FUNCTION_INFO_V1(citus_internal_update_none_dist_table_metadata); +PG_FUNCTION_INFO_V1(citus_internal_database_command); static bool got_SIGTERM = false; @@ -3895,6 +3899,80 @@ citus_internal_update_none_dist_table_metadata(PG_FUNCTION_ARGS) } +/* + * citus_internal_database_command is an internal UDF to + * create/drop a database in an idempotent maner without + * transaction block restrictions. + */ +Datum +citus_internal_database_command(PG_FUNCTION_ARGS) +{ + CheckCitusVersion(ERROR); + if (!ShouldSkipMetadataChecks()) + { + EnsureCoordinatorInitiatedOperation(); + } + PG_ENSURE_ARGNOTNULL(0, "database command"); + + text *commandText = PG_GETARG_TEXT_P(0); + char *command = text_to_cstring(commandText); + Node *parseTree = ParseTreeNode(command); + + int saveNestLevel = NewGUCNestLevel(); + + set_config_option("citus.enable_ddl_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + set_config_option("citus.enable_create_database_propagation", "off", + (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION, + GUC_ACTION_LOCAL, true, 0, false); + + /* + * createdb() / DropDatabase() uses ParseState to report the error position for the + * input command and the position is reported to be 0 when it's provided as NULL. + * We're okay with that because we don't expect this UDF to be called with an incorrect + * DDL command. + */ + ParseState *pstate = NULL; + + if (IsA(parseTree, CreatedbStmt)) + { + CreatedbStmt *stmt = castNode(CreatedbStmt, parseTree); + + bool missingOk = true; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + if (!OidIsValid(databaseOid)) + { + createdb(pstate, (CreatedbStmt *) parseTree); + } + } + else if (IsA(parseTree, DropdbStmt)) + { + DropdbStmt *stmt = castNode(DropdbStmt, parseTree); + + bool missingOk = false; + Oid databaseOid = get_database_oid(stmt->dbname, missingOk); + + + if (OidIsValid(databaseOid)) + { + DropDatabase(pstate, (DropdbStmt *) parseTree); + } + } + else + { + ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); + } + + /* Rollbacks GUCs to the state before this session */ + AtEOXact_GUC(true, saveNestLevel); + + PG_RETURN_VOID(); +} + + /* * SyncNewColocationGroup synchronizes a new pg_dist_colocation entry to a worker. */ @@ -4503,7 +4581,7 @@ PropagateNodeWideObjectsCommandList(void) if (EnableCreateDatabasePropagation) { - /* Get commands for database creation */ + /* get commands for database creation */ List *createDatabaseCommands = GenerateCreateDatabaseCommandList(); ddlCommands = list_concat(ddlCommands, createDatabaseCommands); } diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 32ad4c42725..f06e0f2b0e1 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1266,7 +1266,7 @@ RegisterCitusConfigVariables(void) DefineCustomBoolVariable( "citus.enable_create_database_propagation", gettext_noop("Enables propagating CREATE DATABASE " - "and DROP DATABASE statements to workers"), + "and DROP DATABASE statements to workers."), NULL, &EnableCreateDatabasePropagation, false, diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql index 232e3ad143e..b20f6278e3f 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql @@ -4,7 +4,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) RETURNS void LANGUAGE C - STRICT + VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS 'run a database command without transaction block restrictions'; diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql index 232e3ad143e..b20f6278e3f 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql @@ -4,7 +4,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) RETURNS void LANGUAGE C - STRICT + VOLATILE AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS 'run a database command without transaction block restrictions'; diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 0e43fa386cb..85c55d39a82 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -230,9 +230,6 @@ extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *que ProcessUtilityContext processUtilityContext); -extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); - extern List * PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); @@ -242,10 +239,10 @@ extern List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); -extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool - isPostprocess); +extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, + bool isPostprocess); +extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, + bool isPostprocess); extern List * GenerateCreateDatabaseCommandList(void); @@ -518,7 +515,6 @@ extern List * RenameRoleStmtObjectAddress(Node *stmt, bool missing_ok, bool extern void UnmarkRolesDistributed(List *roles); extern List * FilterDistributedRoles(List *roles); -extern void EnsureSequentialModeForRoleDDL(void); /* schema.c - forward declarations */ extern List * PostprocessCreateSchemaStmt(Node *node, const char *queryString); diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 1790eb46850..5a0b0fed3e8 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -94,7 +94,7 @@ extern void ProcessUtilityParseTree(Node *node, const char *queryString, extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); extern List * DDLTaskList(Oid relationId, const char *commandString); -extern List * NontransactionalNodeDDLTask(TargetWorkerSet targets, List *commands); +extern List * NontransactionalNodeDDLTaskList(TargetWorkerSet targets, List *commands); extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands); extern bool AlterTableInProgress(void); extern bool DropSchemaOrDBInProgress(void); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 66ead2b4d0b..59e6d40c6dd 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -123,12 +123,12 @@ extern void AppendGrantSharedSuffix(StringInfo buf, GrantStmt *stmt); /* Common deparser utils */ -struct option_format +typedef struct DefElemOptionFormat { - const char *name; - const char *format; - const int type; -}; + char *name; + char *format; + int type; +} DefElemOptionFormat; typedef enum OptionFormatType { @@ -140,9 +140,9 @@ typedef enum OptionFormatType } OptionFormatType; -extern void optionToStatement(StringInfo buf, DefElem *option, const struct - option_format *opt_formats, int - opt_formats_len); +extern void DefElemOptionToStatement(StringInfo buf, DefElem *option, const + DefElemOptionFormat *opt_formats, int + opt_formats_len); /* forward declarations for deparse_statistics_stmts.c */ diff --git a/src/include/distributed/metadata/distobject.h b/src/include/distributed/metadata/distobject.h index 86fada5f742..cf24a8c81d3 100644 --- a/src/include/distributed/metadata/distobject.h +++ b/src/include/distributed/metadata/distobject.h @@ -21,13 +21,12 @@ extern bool ObjectExists(const ObjectAddress *address); extern bool CitusExtensionObject(const ObjectAddress *objectAddress); extern bool IsAnyObjectDistributed(const List *addresses); -extern bool IsObjectDistributed(const ObjectAddress *address); extern bool ClusterHasDistributedFunctionWithDistArgument(void); extern void MarkObjectDistributed(const ObjectAddress *distAddress); extern void MarkObjectDistributedViaSuperUser(const ObjectAddress *distAddress); extern void MarkObjectDistributedLocally(const ObjectAddress *distAddress); extern void UnmarkObjectDistributed(const ObjectAddress *address); -extern void UnmarkRolesAndDatabaseDistributed(Node *node); +extern void UnmarkNodeWideObjectsDistributed(Node *node); extern bool IsTableOwnedByExtension(Oid relationId); extern bool ObjectAddressDependsOnExtension(const ObjectAddress *target); extern bool IsAnyObjectAddressOwnedByExtension(const List *targets, diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py index 8f391c44463..b28341e5c39 100755 --- a/src/test/regress/citus_tests/run_test.py +++ b/src/test/regress/citus_tests/run_test.py @@ -151,8 +151,6 @@ def extra_tests(self): ], worker_count=6, ), - "create_drop_database_propagation": TestDeps("minimal_schedule"), - "create_drop_database_propagation_pg15": TestDeps("minimal_schedule"), "function_propagation": TestDeps("minimal_schedule"), "citus_shards": TestDeps("minimal_schedule"), "grant_on_foreign_server_propagation": TestDeps("minimal_schedule"), From ba377ecaabad336b3cc487e6642642a3b0016e1a Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 8 Nov 2023 03:23:34 +0300 Subject: [PATCH 115/180] Fixes tests --- src/test/regress/expected/single_node.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/regress/expected/single_node.out b/src/test/regress/expected/single_node.out index 3b24fd5f55f..522ffb8e804 100644 --- a/src/test/regress/expected/single_node.out +++ b/src/test/regress/expected/single_node.out @@ -90,7 +90,7 @@ SELECT create_distributed_table('failover_to_local', 'a', shard_count=>32); CREATE INDEX CONCURRENTLY ON failover_to_local(a); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: the total number of connections on the server is more than max_connections(100) HINT: Consider using a higher value for max_connections -- reset global GUC changes From 5f8f1d312d6363ae310b4b4fb384a055326874ca Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 8 Nov 2023 06:56:21 +0300 Subject: [PATCH 116/180] Fixes message errors --- .../expected/failure_create_index_concurrently.out | 12 ++++++------ src/test/regress/expected/single_node_0.out | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/regress/expected/failure_create_index_concurrently.out b/src/test/regress/expected/failure_create_index_concurrently.out index 94d0f373dc1..784c91aecf7 100644 --- a/src/test/regress/expected/failure_create_index_concurrently.out +++ b/src/test/regress/expected/failure_create_index_concurrently.out @@ -28,7 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -62,7 +62,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -90,7 +90,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -116,7 +116,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid( CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -143,7 +143,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()'); DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open SELECT citus.mitmproxy('conn.allow()'); mitmproxy @@ -171,7 +171,7 @@ INSERT INTO index_test_2 VALUES (1, 1), (1, 2); CREATE UNIQUE INDEX CONCURRENTLY index_test_2_a_idx ON index_test_2(a); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: could not create unique index "index_test_2_a_idx_1880019" DETAIL: Key (a)=(1) is duplicated. CONTEXT: while executing command on localhost:xxxxx diff --git a/src/test/regress/expected/single_node_0.out b/src/test/regress/expected/single_node_0.out index a44460cca78..12b385e9651 100644 --- a/src/test/regress/expected/single_node_0.out +++ b/src/test/regress/expected/single_node_0.out @@ -90,7 +90,7 @@ SELECT create_distributed_table('failover_to_local', 'a', shard_count=>32); CREATE INDEX CONCURRENTLY ON failover_to_local(a); WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, -if applicable, and then reattempt the original command. +if applicable, and then re-attempt the original command. ERROR: the total number of connections on the server is more than max_connections(100) HINT: Consider using a higher value for max_connections -- reset global GUC changes From 75589d1c119f77dfceb0d21633f592d30efc97c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Wed, 8 Nov 2023 17:38:43 +0300 Subject: [PATCH 117/180] Apply suggestions from code review Co-authored-by: Jelte Fennema-Nio --- src/backend/distributed/commands/database.c | 2 +- .../sql/udfs/citus_internal_database_command/latest.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index cabe69fd6dc..ce429825e1c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -669,7 +669,7 @@ GenerateCreateDatabaseCommandList(void) /* Generate the CREATE DATABASE statement */ appendStringInfo(outerDbStmt, - "SELECT pg_catalog.citus_internal_database_command( %s)", + "SELECT pg_catalog.citus_internal_database_command(%s)", quote_literal_cstr( createStmt)); diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql index b20f6278e3f..f6aaa4642bc 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql @@ -5,6 +5,7 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command te RETURNS void LANGUAGE C VOLATILE + STRICT AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS 'run a database command without transaction block restrictions'; From afa60556fd4e5b3097b7bb89307bff8480bbca12 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 9 Nov 2023 01:43:08 +0300 Subject: [PATCH 118/180] Fixes review comments --- src/backend/distributed/commands/database.c | 38 +++++++++---------- .../latest.sql | 1 - 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index ce429825e1c..ce946777926 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -574,66 +574,64 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) StringInfoData str; initStringInfo(&str); - appendStringInfo(&str, "CREATE DATABASE %s", quote_identifier(NameStr( - databaseForm-> - datname))); + appendStringInfo(&str, "CREATE DATABASE %s", + quote_identifier(NameStr(databaseForm->datname))); if (databaseForm->datdba != InvalidOid) { - appendStringInfo(&str, " OWNER = %s", GetUserNameFromId(databaseForm->datdba, - false)); + appendStringInfo(&str, " OWNER = %s", + quote_literal_cstr(GetUserNameFromId(databaseForm->datdba,false))); } if (databaseForm->encoding != -1) { - appendStringInfo(&str, " ENCODING = '%s'", pg_encoding_to_char( - databaseForm->encoding)); + appendStringInfo(&str, " ENCODING = %s", + quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); } if (collInfo.collation != NULL) { - appendStringInfo(&str, " LC_COLLATE = '%s'", collInfo.collation); + appendStringInfo(&str, " LC_COLLATE = %s", quote_literal_cstr(collInfo.collation)); } if (collInfo.ctype != NULL) { - appendStringInfo(&str, " LC_CTYPE = '%s'", collInfo.ctype); + appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.ctype)); } #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.icu_locale != NULL) { - appendStringInfo(&str, " ICU_LOCALE = '%s'", collInfo.icu_locale); + appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr(collInfo.icu_locale)); } if (databaseForm->datlocprovider != 0) { - appendStringInfo(&str, " LOCALE_PROVIDER = '%s'", GetLocaleProviderString( - databaseForm->datlocprovider)); + appendStringInfo(&str, " LOCALE_PROVIDER = %s", + quote_literal_cstr(GetLocaleProviderString(databaseForm->datlocprovider))); } if (collInfo.collversion != NULL) { - appendStringInfo(&str, " COLLATION_VERSION = '%s'", collInfo.collversion); + appendStringInfo(&str, " COLLATION_VERSION = %s", quote_literal_cstr(collInfo.collversion)); } #endif if (databaseForm->dattablespace != InvalidOid) { - appendStringInfo(&str, " TABLESPACE = %s", quote_identifier(GetTablespaceName( - databaseForm-> - dattablespace))); + appendStringInfo(&str, " TABLESPACE = %s", + quote_identifier(GetTablespaceName(databaseForm->dattablespace))); } - appendStringInfo(&str, " ALLOW_CONNECTIONS = '%s'", databaseForm->datallowconn ? - "true" : "false"); + appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", + quote_literal_cstr(databaseForm->datallowconn ?"true" : "false")); if (databaseForm->datconnlimit >= 0) { appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); } - appendStringInfo(&str, " IS_TEMPLATE = '%s'", databaseForm->datistemplate ? "true" : - "false"); + appendStringInfo(&str, " IS_TEMPLATE = %s", + quote_literal_cstr(databaseForm->datistemplate ? "true" :"false")); FreeDatabaseCollationInfo(collInfo); diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql index f6aaa4642bc..b20f6278e3f 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql @@ -5,7 +5,6 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command te RETURNS void LANGUAGE C VOLATILE - STRICT AS 'MODULE_PATHNAME', $$citus_internal_database_command$$; COMMENT ON FUNCTION pg_catalog.citus_internal_database_command(text) IS 'run a database command without transaction block restrictions'; From 4c9863c0fbc98864656f3eee543212197aff5eb1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 9 Nov 2023 01:49:43 +0300 Subject: [PATCH 119/180] Fixes review comments --- src/backend/distributed/commands/database.c | 1 - src/backend/distributed/deparser/deparse_database_stmts.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index ce946777926..baf6cd42448 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -256,7 +256,6 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, return NIL; } - AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, node); EnsureCoordinator(); diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index e96015fdc1d..a0b9ea7baa7 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -252,7 +252,7 @@ DeparseAlterDatabaseSetStmt(Node *node) /* - * Validates for if option is template, lc_type, locale or lc_collate, propagation will + * Validates for if option is template, lc_type, locale or lc_collate, propagation will * not be supported since template and strategy options are not stored in the catalog * and lc_type, locale and lc_collate options depends on template parameter. */ From 5f4092db5b7c89fb07a041cf1f1a22fdc75b2322 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 9 Nov 2023 12:11:22 +0300 Subject: [PATCH 120/180] Adds validation for template --- .../deparser/deparse_database_stmts.c | 18 ++++---- .../create_drop_database_propagation.out | 40 +++++------------- .../sql/create_drop_database_propagation.sql | 42 +++++-------------- 3 files changed, 32 insertions(+), 68 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index a0b9ea7baa7..a0626372113 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -259,18 +259,20 @@ DeparseAlterDatabaseSetStmt(Node *node) static void ValidateCreateDatabaseOptions(DefElem *option) { - if (strcmp(option->defname, "template") == 0 || - strcmp(option->defname, "strategy") == 0 || - strcmp(option->defname, "lc_ctype") == 0 || - strcmp(option->defname, "locale") == 0 || - strcmp(option->defname, "lc_collate") == 0 || - strcmp(option->defname, "icu_locale") == 0 || - strcmp(option->defname, "locale_provider") == 0) - { + if (strcmp(option->defname, "strategy") == 0){ ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", option->defname)); } + + char *optionValue = defGetString(option); + if (strcmp(option->defname,"template") == 0 && strcmp(optionValue, "template1") != 0) + { + + ereport(ERROR,errmsg("Only template1 is supported as template parameter for CREATE DATABASE")); + + } + } diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 9489664eb42..c5ab0e2dfec 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -12,8 +12,19 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; +-- Tests for create database propagation with template0 which should fail CREATE DATABASE mydatabase WITH OWNER = create_drop_db_test_user + TEMPLATE = 'template0' + ENCODING = 'UTF8' + CONNECTION LIMIT = 10 + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false; +ERROR: Only template1 is supported as template parameter for CREATE DATABASE +CREATE DATABASE mydatabase + WITH template=template1 + OWNER = create_drop_db_test_user ENCODING = 'UTF8' CONNECTION LIMIT = 10 TABLESPACE = create_drop_db_tablespace @@ -239,35 +250,6 @@ NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx ---test for unsupported options -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_CTYPE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; -ERROR: CREATE DATABASE option "lc_ctype" is not supported -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_CTYPE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; -ERROR: CREATE DATABASE option "lc_ctype" is not supported -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_COLLATE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; -ERROR: CREATE DATABASE option "lc_collate" is not supported -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LOCALE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; -ERROR: CREATE DATABASE option "locale" is not supported --clean up resources created by this test drop tablespace create_drop_db_tablespace; \c - - - :worker_1_port diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index ae90088d1b6..157c70b28e0 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -18,9 +18,19 @@ create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; - +-- Tests for create database propagation with template0 which should fail CREATE DATABASE mydatabase WITH OWNER = create_drop_db_test_user + TEMPLATE = 'template0' + ENCODING = 'UTF8' + CONNECTION LIMIT = 10 + TABLESPACE = create_drop_db_tablespace + ALLOW_CONNECTIONS = true + IS_TEMPLATE = false; + +CREATE DATABASE mydatabase + WITH template=template1 + OWNER = create_drop_db_test_user ENCODING = 'UTF8' CONNECTION LIMIT = 10 TABLESPACE = create_drop_db_tablespace @@ -192,36 +202,6 @@ create database "mydatabase#1'2"; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database if exists "mydatabase#1'2"; ---test for unsupported options - -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_CTYPE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; - -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_CTYPE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; - -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LC_COLLATE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; - -CREATE DATABASE mydatabase - with CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - LOCALE = 'C.UTF-8' - ALLOW_CONNECTIONS = false - IS_TEMPLATE = false; - --clean up resources created by this test From 8df30a2a2b04bb4ae81045770b6463538065a593 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 10 Nov 2023 05:52:14 +0300 Subject: [PATCH 121/180] Fixes EnsureCitusInitiatedOperation name change --- src/backend/distributed/metadata/metadata_sync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index d83706535a1..6db3ce28e91 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3904,7 +3904,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) CheckCitusVersion(ERROR); if (!ShouldSkipMetadataChecks()) { - EnsureCoordinatorInitiatedOperation(); + EnsureCitusInitiatedOperation(); } PG_ENSURE_ARGNOTNULL(0, "database command"); From 2ebeea3ce354a6c283f82537d7de1855bf5ff256 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 10 Nov 2023 06:10:24 +0300 Subject: [PATCH 122/180] Fixes compilation warnings --- src/backend/distributed/commands/utility_hook.c | 1 - src/backend/distributed/deparser/citus_deparseutils.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 5d7fa3947c5..eb08b0539bc 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -64,7 +64,6 @@ #include "distributed/multi_executor.h" #include "distributed/multi_explain.h" #include "distributed/multi_physical_planner.h" -#include "distributed/pg_version_constants.h" #include "distributed/reference_table_utils.h" #include "distributed/resource_lock.h" #include "distributed/string_utils.h" diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 0cfd7dd6fcf..bdc591a0162 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -6,11 +6,13 @@ * to their equivalent SQL representation. * */ +#include "pg_version_constants.h" + #include "postgres.h" + #include "commands/defrem.h" #include "distributed/deparser.h" -#include "distributed/pg_version_constants.h" #include "utils/builtins.h" #include "utils/elog.h" #include "utils/rel.h" From 1b9a8ea2db13cab333ff559fb1bb5305fb817e82 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 10 Nov 2023 06:11:43 +0300 Subject: [PATCH 123/180] Fixes indentation --- src/backend/distributed/commands/database.c | 28 +++++++++++-------- .../deparser/deparse_database_stmts.c | 11 ++++---- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index baf6cd42448..b9a459cf4bf 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -574,55 +574,61 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) initStringInfo(&str); appendStringInfo(&str, "CREATE DATABASE %s", - quote_identifier(NameStr(databaseForm->datname))); + quote_identifier(NameStr(databaseForm->datname))); if (databaseForm->datdba != InvalidOid) { appendStringInfo(&str, " OWNER = %s", - quote_literal_cstr(GetUserNameFromId(databaseForm->datdba,false))); + quote_literal_cstr(GetUserNameFromId(databaseForm->datdba, + false))); } if (databaseForm->encoding != -1) { appendStringInfo(&str, " ENCODING = %s", - quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); + quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); } if (collInfo.collation != NULL) { - appendStringInfo(&str, " LC_COLLATE = %s", quote_literal_cstr(collInfo.collation)); + appendStringInfo(&str, " LC_COLLATE = %s", quote_literal_cstr( + collInfo.collation)); } if (collInfo.ctype != NULL) { - appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.ctype)); + appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.ctype)); } #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.icu_locale != NULL) { - appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr(collInfo.icu_locale)); + appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr( + collInfo.icu_locale)); } if (databaseForm->datlocprovider != 0) { appendStringInfo(&str, " LOCALE_PROVIDER = %s", - quote_literal_cstr(GetLocaleProviderString(databaseForm->datlocprovider))); + quote_literal_cstr(GetLocaleProviderString( + databaseForm->datlocprovider))); } if (collInfo.collversion != NULL) { - appendStringInfo(&str, " COLLATION_VERSION = %s", quote_literal_cstr(collInfo.collversion)); + appendStringInfo(&str, " COLLATION_VERSION = %s", quote_literal_cstr( + collInfo.collversion)); } #endif if (databaseForm->dattablespace != InvalidOid) { appendStringInfo(&str, " TABLESPACE = %s", - quote_identifier(GetTablespaceName(databaseForm->dattablespace))); + quote_identifier(GetTablespaceName( + databaseForm->dattablespace))); } appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", - quote_literal_cstr(databaseForm->datallowconn ?"true" : "false")); + quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); if (databaseForm->datconnlimit >= 0) { @@ -630,7 +636,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) } appendStringInfo(&str, " IS_TEMPLATE = %s", - quote_literal_cstr(databaseForm->datistemplate ? "true" :"false")); + quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); FreeDatabaseCollationInfo(collInfo); diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index a0626372113..494bfb30cba 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -259,20 +259,19 @@ DeparseAlterDatabaseSetStmt(Node *node) static void ValidateCreateDatabaseOptions(DefElem *option) { - if (strcmp(option->defname, "strategy") == 0){ + if (strcmp(option->defname, "strategy") == 0) + { ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", option->defname)); } char *optionValue = defGetString(option); - if (strcmp(option->defname,"template") == 0 && strcmp(optionValue, "template1") != 0) + if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, "template1") != 0) { - - ereport(ERROR,errmsg("Only template1 is supported as template parameter for CREATE DATABASE")); - + ereport(ERROR, errmsg( + "Only template1 is supported as template parameter for CREATE DATABASE")); } - } From 7a6afb0beb1dd10b8d7e00085c833bce51edfe06 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 10 Nov 2023 07:24:54 +0300 Subject: [PATCH 124/180] Fixes review comments --- .../regress/expected/create_drop_database_propagation.out | 2 ++ src/test/regress/sql/create_drop_database_propagation.sql | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index c5ab0e2dfec..3cd59dc7516 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -114,6 +114,8 @@ select 1 from citus_add_node('localhost', :worker_2_port); 1 (1 row) +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 157c70b28e0..09386c9b4bf 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -55,8 +55,6 @@ SELECT result from run_command_on_all_nodes( drop database mydatabase; - - SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( @@ -103,6 +101,9 @@ SELECT result from run_command_on_all_nodes( select 1 from citus_add_node('localhost', :worker_2_port); +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE DATABASE%'; + SELECT result from run_command_on_all_nodes( $$ SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( From 3067d1ef083b79b42f5b8d3115cdb436279eed1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Fri, 10 Nov 2023 17:34:09 +0300 Subject: [PATCH 125/180] Apply suggestions from code review Co-authored-by: Onur Tirtir --- src/backend/distributed/commands/database.c | 6 ++++-- .../distributed/deparser/citus_deparseutils.c | 18 +++++++++++------- src/backend/distributed/metadata/distobject.c | 6 ++++-- .../distributed/metadata/metadata_sync.c | 6 ++++-- src/include/distributed/deparser.h | 6 +++--- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index b9a459cf4bf..0f4705c21ff 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -275,7 +275,7 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, * postgres instance. * * In this stage, we can perform validations and prepare the commands that need to - * be run on all workers to grant. + * be run on all workers to create the database. */ List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, @@ -298,7 +298,9 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, /* * PostprocessCreatedbStmt is executed after the statement is applied to the local * postgres instance. In this stage we can prepare the commands that need to be run on - * all workers to create the database. + * all workers to create the database. Since the CREATE DATABASE statement gives error + * in a transaction block, we need to use NontransactionalNodeDDLTaskList to send the + * CREATE DATABASE statement to the workers. * */ List * diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index bdc591a0162..1fae9407f93 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -1,11 +1,15 @@ -/* +/*------------------------------------------------------------------------- + * * citus_deparseutils.c - * --------------------- * - * This file contains common functions used for deparsing PostgreSQL statements - * to their equivalent SQL representation. + * This file contains common functions used for deparsing PostgreSQL + * statements to their equivalent SQL representation. + * + * Copyright (c) Citus Data, Inc. * + *------------------------------------------------------------------------- */ + #include "pg_version_constants.h" #include "postgres.h" @@ -31,9 +35,9 @@ * @param optionFormatsLen The number of option formats in the opt_formats array. */ void -DefElemOptionToStatement(StringInfo buf, DefElem *option, const - DefElemOptionFormat *optionFormats, int - optionFormatsLen) +DefElemOptionToStatement(StringInfo buf, DefElem *option, + const DefElemOptionFormat *optionFormats, + int optionFormatsLen) { const char *name = option->defname; int i; diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index afdaf57c18e..5350a08bc94 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -360,8 +360,10 @@ ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, /* - * Deletes all pg_dist_object records for distributed roles in `DROP ROLE` statement a - * and for all databases in `DROP DATABASE` statement + * UnmarkNodeWideObjectsDistributed deletes pg_dist_object records + * for all distributed objects in given Drop stmt node. + * + * Today we only expect DropRoleStmt and DropdbStmt to get here. */ void UnmarkNodeWideObjectsDistributed(Node *node) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 6db3ce28e91..dfcf243bf3f 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3902,11 +3902,13 @@ Datum citus_internal_database_command(PG_FUNCTION_ARGS) { CheckCitusVersion(ERROR); + if (!ShouldSkipMetadataChecks()) { EnsureCitusInitiatedOperation(); } - PG_ENSURE_ARGNOTNULL(0, "database command"); + + PG_ENSURE_ARGNOTNULL(0, "command"); text *commandText = PG_GETARG_TEXT_P(0); char *command = text_to_cstring(commandText); @@ -3960,7 +3962,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); } - /* Rollbacks GUCs to the state before this session */ + /* rollback GUCs to the state before this session */ AtEOXact_GUC(true, saveNestLevel); PG_RETURN_VOID(); diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 59e6d40c6dd..5159f4c9d1e 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -140,9 +140,9 @@ typedef enum OptionFormatType } OptionFormatType; -extern void DefElemOptionToStatement(StringInfo buf, DefElem *option, const - DefElemOptionFormat *opt_formats, int - opt_formats_len); +extern void DefElemOptionToStatement(StringInfo buf, DefElem *option, + const DefElemOptionFormat *opt_formats, + int opt_formats_len); /* forward declarations for deparse_statistics_stmts.c */ From f8b3f322aae794128cd808fe8919e5ad490c9fd1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Fri, 10 Nov 2023 08:33:51 +0300 Subject: [PATCH 126/180] Fixed review items --- src/backend/distributed/commands/database.c | 46 ++++--------------- .../distributed/commands/utility_hook.c | 1 - .../distributed/deparser/citus_deparseutils.c | 1 - .../deparser/deparse_database_stmts.c | 6 --- src/backend/distributed/metadata/distobject.c | 5 +- .../distributed/metadata/metadata_sync.c | 1 - 6 files changed, 11 insertions(+), 49 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 0f4705c21ff..2d0a2ce1692 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -298,7 +298,7 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, /* * PostprocessCreatedbStmt is executed after the statement is applied to the local * postgres instance. In this stage we can prepare the commands that need to be run on - * all workers to create the database. Since the CREATE DATABASE statement gives error + * all workers to create the database. Since the CREATE DATABASE statement gives error * in a transaction block, we need to use NontransactionalNodeDDLTaskList to send the * CREATE DATABASE statement to the workers. * @@ -388,11 +388,11 @@ GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) * object of the DropdbStmt. */ List * -DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) +DropDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { DropdbStmt *stmt = castNode(DropdbStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, - missing_ok); + missingOk); return list_make1(dbAddress); } @@ -402,11 +402,11 @@ DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) * object of the CreatedbStmt. */ List * -CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) +CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) { CreatedbStmt *stmt = castNode(CreatedbStmt, node); ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(stmt->dbname, - missing_ok); + missingOk); return list_make1(dbAddress); } @@ -424,7 +424,7 @@ GetTablespaceName(Oid tablespaceOid) } Form_pg_tablespace tablespaceForm = (Form_pg_tablespace) GETSTRUCT(tuple); - char *tablespaceName = NameStr(tablespaceForm->spcname); + char *tablespaceName = pstrdup(NameStr(tablespaceForm->spcname)); ReleaseSysCache(tuple); @@ -437,17 +437,17 @@ GetTablespaceName(Oid tablespaceOid) * We need this method since collation related info in Form_pg_database is not accessible */ static DatabaseCollationInfo -GetDatabaseCollation(Oid db_oid) +GetDatabaseCollation(Oid dbOid) { DatabaseCollationInfo info; bool isNull; Snapshot snapshot = RegisterSnapshot(GetLatestSnapshot()); Relation rel = table_open(DatabaseRelationId, AccessShareLock); - HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, db_oid); + HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, dbOid); if (!HeapTupleIsValid(tup)) { - elog(ERROR, "cache lookup failed for database %u", db_oid); + elog(ERROR, "cache lookup failed for database %u", dbOid); } TupleDesc tupdesc = RelationGetDescr(rel); @@ -505,29 +505,6 @@ GetDatabaseCollation(Oid db_oid) } -/* - * FreeDatabaseCollationInfo frees the memory allocated for DatabaseCollationInfo - */ -static void -FreeDatabaseCollationInfo(DatabaseCollationInfo collInfo) -{ - if (collInfo.collation != NULL) - { - pfree(collInfo.collation); - } - if (collInfo.ctype != NULL) - { - pfree(collInfo.ctype); - } - #if PG_VERSION_NUM >= PG_VERSION_15 - if (collInfo.icu_locale != NULL) - { - pfree(collInfo.icu_locale); - } - #endif -} - - #if PG_VERSION_NUM >= PG_VERSION_15 /* @@ -640,8 +617,6 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " IS_TEMPLATE = %s", quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); - FreeDatabaseCollationInfo(collInfo); - return str.data; } @@ -669,7 +644,6 @@ GenerateCreateDatabaseCommandList(void) char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); - StringInfo outerDbStmt = makeStringInfo(); /* Generate the CREATE DATABASE statement */ @@ -678,8 +652,6 @@ GenerateCreateDatabaseCommandList(void) quote_literal_cstr( createStmt)); - elog(LOG, "outerDbStmt: %s", outerDbStmt->data); - /* Add the statement to the list of commands */ commands = lappend(commands, outerDbStmt->data); } diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index eb08b0539bc..29d7e08da1f 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -25,7 +25,6 @@ *------------------------------------------------------------------------- */ - #include "pg_version_constants.h" #include "postgres.h" diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 1fae9407f93..3f83f57fdfc 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -14,7 +14,6 @@ #include "postgres.h" - #include "commands/defrem.h" #include "distributed/deparser.h" #include "utils/builtins.h" diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 494bfb30cba..bba6bedb4bc 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -275,9 +275,6 @@ ValidateCreateDatabaseOptions(DefElem *option) } -/* - * Prepares a CREATE DATABASE statement with given empty StringInfo buffer and CreatedbStmt node. - */ static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -314,9 +311,6 @@ DeparseCreateDatabaseStmt(Node *node) } -/* - * Prepares a DROP DATABASE statement with given empty StringInfo buffer and DropdbStmt node. - */ static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) { diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 5350a08bc94..94c12d47f5d 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -22,11 +22,13 @@ #include "catalog/dependency.h" #include "catalog/namespace.h" #include "catalog/objectaddress.h" +#include "catalog/pg_database.h" #include "catalog/pg_extension_d.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "citus_version.h" +#include "commands/dbcommands.h" #include "commands/extension.h" #include "distributed/listutils.h" #include "distributed/colocation_utils.h" @@ -48,9 +50,6 @@ #include "utils/lsyscache.h" #include "utils/regproc.h" #include "utils/rel.h" -#include "catalog/pg_database.h" -#include "commands/dbcommands.h" - static char * CreatePgDistObjectEntryCommand(const ObjectAddress *objectAddress); static int ExecuteCommandAsSuperuser(char *query, int paramCount, Oid *paramTypes, diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index dfcf243bf3f..e612a468ad4 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3951,7 +3951,6 @@ citus_internal_database_command(PG_FUNCTION_ARGS) bool missingOk = false; Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - if (OidIsValid(databaseOid)) { DropDatabase(pstate, (DropdbStmt *) parseTree); From fe242276387cc7c9bbb1924c44f2c439a92262f7 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 10 Nov 2023 22:00:41 +0300 Subject: [PATCH 127/180] Improve tests for PG <= 14 --- .../create_drop_database_propagation.out | 500 +++++++++++++----- .../regress/expected/multi_test_helpers.out | 54 ++ .../sql/create_drop_database_propagation.sql | 272 ++++++---- src/test/regress/sql/multi_test_helpers.sql | 56 ++ 4 files changed, 638 insertions(+), 244 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 3cd59dc7516..6bc94e7a199 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -1,6 +1,58 @@ --- test for create/drop database propagation --- This test is only executes for Postgres 14 --- For postgres 15 tests, pg15_create_drop_database_propagation.sql is used +-- Test for create/drop database propagation. +-- This test is only executes for Postgres versions < 15. +-- For versions >= 15, pg15_create_drop_database_propagation.sql is used. +-- For versions >= 16, pg16_create_drop_database_propagation.sql is used. +-- Test the UDF that we use to issue database command during metadata sync. +SELECT pg_catalog.citus_internal_database_command(null); +ERROR: This is an internal Citus function can only be used in a distributed transaction +CREATE ROLE test_db_commands WITH LOGIN; +ALTER SYSTEM SET citus.enable_manual_metadata_changes_for_user TO 'test_db_commands'; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + +SET ROLE test_db_commands; +-- fails on null input +SELECT pg_catalog.citus_internal_database_command(null); +ERROR: command cannot be NULL +-- fails on non create / drop db command +SELECT pg_catalog.citus_internal_database_command('CREATE TABLE foo_bar(a int)'); +ERROR: unsupported command type 255 +SELECT pg_catalog.citus_internal_database_command('SELECT 1'); +ERROR: unsupported command type 242 +SELECT pg_catalog.citus_internal_database_command('asfsfdsg'); +ERROR: syntax error at or near "asfsfdsg" +SELECT pg_catalog.citus_internal_database_command(''); +ERROR: cannot execute multiple utility events +RESET ROLE; +ALTER ROLE test_db_commands nocreatedb; +SET ROLE test_db_commands; +-- make sure that pg_catalog.citus_internal_database_command doesn't cause privilege escalation +SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE no_permissions'); +ERROR: permission denied to create database +RESET ROLE; +DROP USER test_db_commands; +ALTER SYSTEM RESET citus.enable_manual_metadata_changes_for_user; +SELECT pg_reload_conf(); + pg_reload_conf +--------------------------------------------------------------------- + t +(1 row) + +SELECT pg_sleep(0.1); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; \c - - - :worker_1_port @@ -9,6 +61,54 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace \c - - - :worker_2_port \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port +CREATE DATABASE local_database; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +-- check that it's only created for coordinator +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +DROP DATABASE local_database; +-- and is dropped +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +\c - - - :worker_1_port +CREATE DATABASE local_database; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +-- check that it's only created for coordinator +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +DROP DATABASE local_database; +-- and is dropped +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + \c - - - :master_port create user create_drop_db_test_user; set citus.enable_create_database_propagation=on; @@ -22,7 +122,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; ERROR: Only template1 is supported as template parameter for CREATE DATABASE -CREATE DATABASE mydatabase +CREATE DATABASE mydatabase_1 WITH template=template1 OWNER = create_drop_db_test_user ENCODING = 'UTF8' @@ -30,47 +130,104 @@ CREATE DATABASE mydatabase TABLESPACE = create_drop_db_tablespace ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT result from run_command_on_all_nodes( +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +-- Test LC / LOCALE settings that don't match the ones provided in template db. +-- All should throw an error on the coordinator. +CREATE DATABASE lc_collate_test LC_COLLATE = 'en_US.UTF-8'; +ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +HINT: Use the same collation as in the template database, or use template0 as template. +CREATE DATABASE lc_ctype_test LC_CTYPE = 'en_US.UTF-8'; +ERROR: new LC_CTYPE (en_US.UTF-8) is incompatible with the LC_CTYPE of the template database (C) +HINT: Use the same LC_CTYPE as in the template database, or use template0 as template. +CREATE DATABASE locale_test LOCALE = 'en_US.UTF-8'; +ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +HINT: Use the same collation as in the template database, or use template0 as template. +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'; +ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +HINT: Use the same collation as in the template database, or use template0 as template. +-- Test LC / LOCALE settings that match the ones provided in template db. +CREATE DATABASE lc_collate_test LC_COLLATE = 'C'; +CREATE DATABASE lc_ctype_test LC_CTYPE = 'C'; +CREATE DATABASE locale_test LOCALE = 'C'; +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'C' LC_CTYPE = 'C'; +SELECT * FROM public.check_database_on_all_nodes('lc_collate_test') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('lc_ctype_test') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('locale_test') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('lc_collate_lc_ctype_test') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +DROP DATABASE lc_collate_test; +DROP DATABASE lc_ctype_test; +DROP DATABASE locale_test; +DROP DATABASE lc_collate_lc_ctype_test; +-- ALTER TABLESPACE .. RENAME TO .. is not supported, so we need to rename it manually. +SELECT result FROM run_command_on_all_nodes( $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 + ALTER TABLESPACE create_drop_db_tablespace RENAME TO "ts-needs\!escape" $$ -) ORDER BY result; - result +); + result --------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + ALTER TABLESPACE + ALTER TABLESPACE + ALTER TABLESPACE (3 rows) -drop database mydatabase; -SELECT result from run_command_on_all_nodes( +CREATE USER "role-needs\!escape"; +CREATE DATABASE "db-needs\!escape" owner "role-needs\!escape" tablespace "ts-needs\!escape"; +-- Rename it to make check_database_on_all_nodes happy. +-- Today we don't support ALTER DATABASE .. RENAME TO .., so need to propagate it manually. +SELECT result FROM run_command_on_all_nodes( $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 + ALTER DATABASE "db-needs\!escape" RENAME TO db_needs_escape $$ -) ORDER BY result; - result +); + result --------------------------------------------------------------------- + ALTER DATABASE + ALTER DATABASE + ALTER DATABASE +(3 rows) - - +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -- test database syncing after node addition @@ -85,56 +242,128 @@ CREATE DATABASE mydatabase OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace + TABLESPACE = "ts-needs\!escape" ALLOW_CONNECTIONS = false IS_TEMPLATE = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (2 rows) +SET citus.metadata_sync_mode to 'transactional'; select 1 from citus_add_node('localhost', :worker_2_port); ?column? --------------------------------------------------------------------- 1 (1 row) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SET citus.metadata_sync_mode to 'nontransactional'; +select 1 from citus_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +RESET citus.metadata_sync_mode; +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT citus_disable_node_and_wait('localhost', :worker_1_port, true); + citus_disable_node_and_wait +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE test_node_activation; +SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +SELECT * FROM public.check_database_on_all_nodes('test_node_activation') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SET citus.log_remote_commands = true; @@ -145,54 +374,37 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE mydatabase DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result +-- check that we actually drop the database +drop database mydatabase_1; +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- + coordinator (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) - - +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace + TABLESPACE = "ts-needs\!escape" ALLOW_CONNECTIONS = false IS_TEMPLATE = true; -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result +SELECT * FROM public.check_database_on_all_nodes('my_template_database') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] + coordinator (local) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) --template databases could not be dropped so we need to change the template flag @@ -216,25 +428,12 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE my_template_database DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result +SELECT * FROM public.check_database_on_all_nodes('my_template_database') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - - - + coordinator (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) --tests for special characters in database name @@ -252,11 +451,44 @@ NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx ---clean up resources created by this test -drop tablespace create_drop_db_tablespace; +reset citus.grep_remote_commands; +reset citus.log_remote_commands; +-- it doesn't fail thanks to "if exists" +drop database if exists "mydatabase#1'2"; +NOTICE: database "mydatabase#1'2" does not exist, skipping +-- recreate it to verify that it's actually dropped +create database "mydatabase#1'2"; +drop database "mydatabase#1'2"; +-- second time we try to drop it, it fails due to lack of "if exists" +drop database "mydatabase#1'2"; +ERROR: database "mydatabase#1'2" does not exist \c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; +SET citus.enable_create_database_propagation TO ON; +-- show that dropping the database from workers is not allowed when citus.enable_create_database_propagation is on +DROP DATABASE db_needs_escape; +ERROR: operation is not allowed on this node +HINT: Connect to the coordinator and run it again. +-- and the same applies to create database too +create database error_test; +ERROR: operation is not allowed on this node +HINT: Connect to the coordinator and run it again. \c - - - :master_port +SET citus.enable_create_database_propagation TO ON; +DROP DATABASE test_node_activation; +DROP DATABASE db_needs_escape; +DROP USER "role-needs\!escape"; +--clean up resources created by this test +-- DROP TABLESPACE is not supported, so we need to drop it manually. +SELECT result FROM run_command_on_all_nodes( + $$ + drop tablespace "ts-needs\!escape" + $$ +); + result +--------------------------------------------------------------------- + DROP TABLESPACE + DROP TABLESPACE + DROP TABLESPACE +(3 rows) + drop user create_drop_db_test_user; diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index b8758e561bd..da771b4c5eb 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -526,3 +526,57 @@ BEGIN RETURN result; END; $func$ LANGUAGE plpgsql; +-- For all nodes, returns database properties of given database, except +-- oid, datfrozenxid and datminmxid. +-- +-- Also returns whether the node has a pg_dist_object record for the database +-- and whether there are any stale pg_dist_object records for a database. +CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) +RETURNS TABLE (node_type text, result text) +AS $func$ +BEGIN + RETURN QUERY + SELECT + CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' + WHEN (groupid = 0) THEN 'coordinator (remote)' + WHEN (groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'worker node (local)' + ELSE 'worker node (remote)' + END AS node_type, + q2.result + FROM run_command_on_all_nodes( + format( + $$ + SELECT to_jsonb(q.*) + FROM ( + SELECT + ( + SELECT to_jsonb(database_properties.*) + FROM ( + SELECT datname, pa.rolname as database_owner, + pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, + datistemplate, datallowconn, datconnlimit, + pt.spcname AS tablespace, datcollate, datctype, daticulocale, + datcollversion, datacl + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + JOIN pg_tablespace pt ON pd.dattablespace = pt.oid + WHERE datname = '%s' + ) database_properties + ) AS database_properties, + ( + SELECT COUNT(*)=1 + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + ) AS pg_dist_object_record_for_db_exists, + ( + SELECT COUNT(*) > 0 + FROM pg_dist_object + WHERE classid = 1262 AND objid NOT IN (SELECT oid FROM pg_database) + ) AS stale_pg_dist_object_record_for_a_db_exists + ) q + $$, + p_database_name, p_database_name + ) + ) q2 + JOIN pg_dist_node USING (nodeid); +END; +$func$ LANGUAGE plpgsql; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 09386c9b4bf..d75ecdf9fb0 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -1,7 +1,40 @@ --- test for create/drop database propagation --- This test is only executes for Postgres 14 --- For postgres 15 tests, pg15_create_drop_database_propagation.sql is used +-- Test for create/drop database propagation. +-- This test is only executes for Postgres versions < 15. +-- For versions >= 15, pg15_create_drop_database_propagation.sql is used. +-- For versions >= 16, pg16_create_drop_database_propagation.sql is used. + +-- Test the UDF that we use to issue database command during metadata sync. +SELECT pg_catalog.citus_internal_database_command(null); + +CREATE ROLE test_db_commands WITH LOGIN; +ALTER SYSTEM SET citus.enable_manual_metadata_changes_for_user TO 'test_db_commands'; +SELECT pg_reload_conf(); +SELECT pg_sleep(0.1); +SET ROLE test_db_commands; + +-- fails on null input +SELECT pg_catalog.citus_internal_database_command(null); + +-- fails on non create / drop db command +SELECT pg_catalog.citus_internal_database_command('CREATE TABLE foo_bar(a int)'); +SELECT pg_catalog.citus_internal_database_command('SELECT 1'); +SELECT pg_catalog.citus_internal_database_command('asfsfdsg'); +SELECT pg_catalog.citus_internal_database_command(''); + +RESET ROLE; +ALTER ROLE test_db_commands nocreatedb; +SET ROLE test_db_commands; + +-- make sure that pg_catalog.citus_internal_database_command doesn't cause privilege escalation +SELECT pg_catalog.citus_internal_database_command('CREATE DATABASE no_permissions'); + +RESET ROLE; +DROP USER test_db_commands; +ALTER SYSTEM RESET citus.enable_manual_metadata_changes_for_user; +SELECT pg_reload_conf(); +SELECT pg_sleep(0.1); + \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; @@ -13,6 +46,28 @@ CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace \set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +\c - - - :master_port +CREATE DATABASE local_database; + +-- check that it's only created for coordinator +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + +DROP DATABASE local_database; + +-- and is dropped +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + +\c - - - :worker_1_port +CREATE DATABASE local_database; + +-- check that it's only created for coordinator +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + +DROP DATABASE local_database; + +-- and is dropped +SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; + \c - - - :master_port create user create_drop_db_test_user; @@ -28,7 +83,7 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -CREATE DATABASE mydatabase +CREATE DATABASE mydatabase_1 WITH template=template1 OWNER = create_drop_db_test_user ENCODING = 'UTF8' @@ -37,38 +92,51 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = true IS_TEMPLATE = false; -SELECT result from run_command_on_all_nodes( +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + +-- Test LC / LOCALE settings that don't match the ones provided in template db. +-- All should throw an error on the coordinator. +CREATE DATABASE lc_collate_test LC_COLLATE = 'en_US.UTF-8'; +CREATE DATABASE lc_ctype_test LC_CTYPE = 'en_US.UTF-8'; +CREATE DATABASE locale_test LOCALE = 'en_US.UTF-8'; +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'; + +-- Test LC / LOCALE settings that match the ones provided in template db. +CREATE DATABASE lc_collate_test LC_COLLATE = 'C'; +CREATE DATABASE lc_ctype_test LC_CTYPE = 'C'; +CREATE DATABASE locale_test LOCALE = 'C'; +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'C' LC_CTYPE = 'C'; + +SELECT * FROM public.check_database_on_all_nodes('lc_collate_test') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('lc_ctype_test') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('locale_test') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('lc_collate_lc_ctype_test') ORDER BY node_type; + +DROP DATABASE lc_collate_test; +DROP DATABASE lc_ctype_test; +DROP DATABASE locale_test; +DROP DATABASE lc_collate_lc_ctype_test; + +-- ALTER TABLESPACE .. RENAME TO .. is not supported, so we need to rename it manually. +SELECT result FROM run_command_on_all_nodes( $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 + ALTER TABLESPACE create_drop_db_tablespace RENAME TO "ts-needs\!escape" $$ -) ORDER BY result; +); +CREATE USER "role-needs\!escape"; -drop database mydatabase; +CREATE DATABASE "db-needs\!escape" owner "role-needs\!escape" tablespace "ts-needs\!escape"; -SELECT result from run_command_on_all_nodes( +-- Rename it to make check_database_on_all_nodes happy. +-- Today we don't support ALTER DATABASE .. RENAME TO .., so need to propagate it manually. +SELECT result FROM run_command_on_all_nodes( $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 + ALTER DATABASE "db-needs\!escape" RENAME TO db_needs_escape $$ -) ORDER BY result; +); + +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; -- test database syncing after node addition @@ -79,45 +147,39 @@ CREATE DATABASE mydatabase OWNER = create_drop_db_test_user CONNECTION LIMIT = 10 ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace + TABLESPACE = "ts-needs\!escape" ALLOW_CONNECTIONS = false IS_TEMPLATE = false; +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; +SET citus.metadata_sync_mode to 'transactional'; +select 1 from citus_add_node('localhost', :worker_2_port); + +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; +select 1 from citus_remove_node('localhost', :worker_2_port); + +SET citus.metadata_sync_mode to 'nontransactional'; select 1 from citus_add_node('localhost', :worker_2_port); -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; +RESET citus.metadata_sync_mode; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; + +SELECT citus_disable_node_and_wait('localhost', :worker_1_port, true); + +CREATE DATABASE test_node_activation; +SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); + +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; +SELECT * FROM public.check_database_on_all_nodes('test_node_activation') ORDER BY node_type; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%DROP DATABASE%'; @@ -125,45 +187,22 @@ drop database mydatabase; SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; +-- check that we actually drop the database +drop database mydatabase_1; + +SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; + +SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; -- create a template database with all options set and allow connections false CREATE DATABASE my_template_database WITH OWNER = create_drop_db_test_user ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace + TABLESPACE = "ts-needs\!escape" ALLOW_CONNECTIONS = false IS_TEMPLATE = true; -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; +SELECT * FROM public.check_database_on_all_nodes('my_template_database') ORDER BY node_type; --template databases could not be dropped so we need to change the template flag SELECT result from run_command_on_all_nodes( @@ -178,20 +217,8 @@ set citus.grep_remote_commands = '%DROP DATABASE%'; drop database my_template_database; SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; + +SELECT * FROM public.check_database_on_all_nodes('my_template_database') ORDER BY node_type; --tests for special characters in database name set citus.enable_create_database_propagation=on; @@ -203,19 +230,44 @@ create database "mydatabase#1'2"; set citus.grep_remote_commands = '%DROP DATABASE%'; drop database if exists "mydatabase#1'2"; +reset citus.grep_remote_commands; +reset citus.log_remote_commands; ---clean up resources created by this test +-- it doesn't fail thanks to "if exists" +drop database if exists "mydatabase#1'2"; + +-- recreate it to verify that it's actually dropped +create database "mydatabase#1'2"; +drop database "mydatabase#1'2"; -drop tablespace create_drop_db_tablespace; +-- second time we try to drop it, it fails due to lack of "if exists" +drop database "mydatabase#1'2"; \c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; +SET citus.enable_create_database_propagation TO ON; -\c - - - :worker_2_port +-- show that dropping the database from workers is not allowed when citus.enable_create_database_propagation is on +DROP DATABASE db_needs_escape; -drop tablespace create_drop_db_tablespace; +-- and the same applies to create database too +create database error_test; \c - - - :master_port +SET citus.enable_create_database_propagation TO ON; + +DROP DATABASE test_node_activation; +DROP DATABASE db_needs_escape; +DROP USER "role-needs\!escape"; + +--clean up resources created by this test + +-- DROP TABLESPACE is not supported, so we need to drop it manually. +SELECT result FROM run_command_on_all_nodes( + $$ + drop tablespace "ts-needs\!escape" + $$ +); + drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index f7c97f1b2d3..2e8654ce201 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -550,3 +550,59 @@ BEGIN RETURN result; END; $func$ LANGUAGE plpgsql; + + +-- For all nodes, returns database properties of given database, except +-- oid, datfrozenxid and datminmxid. +-- +-- Also returns whether the node has a pg_dist_object record for the database +-- and whether there are any stale pg_dist_object records for a database. +CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) +RETURNS TABLE (node_type text, result text) +AS $func$ +BEGIN + RETURN QUERY + SELECT + CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' + WHEN (groupid = 0) THEN 'coordinator (remote)' + WHEN (groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'worker node (local)' + ELSE 'worker node (remote)' + END AS node_type, + q2.result + FROM run_command_on_all_nodes( + format( + $$ + SELECT to_jsonb(q.*) + FROM ( + SELECT + ( + SELECT to_jsonb(database_properties.*) + FROM ( + SELECT datname, pa.rolname as database_owner, + pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, + datistemplate, datallowconn, datconnlimit, + pt.spcname AS tablespace, datcollate, datctype, daticulocale, + datcollversion, datacl + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + JOIN pg_tablespace pt ON pd.dattablespace = pt.oid + WHERE datname = '%s' + ) database_properties + ) AS database_properties, + ( + SELECT COUNT(*)=1 + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + ) AS pg_dist_object_record_for_db_exists, + ( + SELECT COUNT(*) > 0 + FROM pg_dist_object + WHERE classid = 1262 AND objid NOT IN (SELECT oid FROM pg_database) + ) AS stale_pg_dist_object_record_for_a_db_exists + ) q + $$, + p_database_name, p_database_name + ) + ) q2 + JOIN pg_dist_node USING (nodeid); +END; +$func$ LANGUAGE plpgsql; From 5b446b11372043ccb5648b561ef8816dfc0f5635 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 11:27:29 +0300 Subject: [PATCH 128/180] make tests passing --- .../distributed/metadata/metadata_sync.c | 4 +++- .../create_drop_database_propagation.out | 20 +++++++++---------- .../sql/create_drop_database_propagation.sql | 8 ++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index e612a468ad4..0c04502e6dc 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3958,7 +3958,9 @@ citus_internal_database_command(PG_FUNCTION_ARGS) } else { - ereport(ERROR, (errmsg("unsupported command type %d", nodeTag(parseTree)))); + ereport(ERROR, (errmsg("citus_internal_database_command() can only be used " + "for CREATE DATABASE and DROP DATABASE commands by " + "Citus."))); } /* rollback GUCs to the state before this session */ diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 6bc94e7a199..00f95c7c306 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -25,9 +25,9 @@ SELECT pg_catalog.citus_internal_database_command(null); ERROR: command cannot be NULL -- fails on non create / drop db command SELECT pg_catalog.citus_internal_database_command('CREATE TABLE foo_bar(a int)'); -ERROR: unsupported command type 255 +ERROR: citus_internal_database_command() can only be used for CREATE DATABASE and DROP DATABASE commands by Citus. SELECT pg_catalog.citus_internal_database_command('SELECT 1'); -ERROR: unsupported command type 242 +ERROR: citus_internal_database_command() can only be used for CREATE DATABASE and DROP DATABASE commands by Citus. SELECT pg_catalog.citus_internal_database_command('asfsfdsg'); ERROR: syntax error at or near "asfsfdsg" SELECT pg_catalog.citus_internal_database_command(''); @@ -140,17 +140,17 @@ SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_t -- Test LC / LOCALE settings that don't match the ones provided in template db. -- All should throw an error on the coordinator. -CREATE DATABASE lc_collate_test LC_COLLATE = 'en_US.UTF-8'; -ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +CREATE DATABASE lc_collate_test LC_COLLATE = 'C.UTF-8'; +ERROR: new collation (C.UTF-8) is incompatible with the collation of the template database (C) HINT: Use the same collation as in the template database, or use template0 as template. -CREATE DATABASE lc_ctype_test LC_CTYPE = 'en_US.UTF-8'; -ERROR: new LC_CTYPE (en_US.UTF-8) is incompatible with the LC_CTYPE of the template database (C) +CREATE DATABASE lc_ctype_test LC_CTYPE = 'C.UTF-8'; +ERROR: new LC_CTYPE (C.UTF-8) is incompatible with the LC_CTYPE of the template database (C) HINT: Use the same LC_CTYPE as in the template database, or use template0 as template. -CREATE DATABASE locale_test LOCALE = 'en_US.UTF-8'; -ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +CREATE DATABASE locale_test LOCALE = 'C.UTF-8'; +ERROR: new collation (C.UTF-8) is incompatible with the collation of the template database (C) HINT: Use the same collation as in the template database, or use template0 as template. -CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'; -ERROR: new collation (en_US.UTF-8) is incompatible with the collation of the template database (C) +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'C.UTF-8' LC_CTYPE = 'C.UTF-8'; +ERROR: new collation (C.UTF-8) is incompatible with the collation of the template database (C) HINT: Use the same collation as in the template database, or use template0 as template. -- Test LC / LOCALE settings that match the ones provided in template db. CREATE DATABASE lc_collate_test LC_COLLATE = 'C'; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index d75ecdf9fb0..6e6ecacb5ba 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -96,10 +96,10 @@ SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_t -- Test LC / LOCALE settings that don't match the ones provided in template db. -- All should throw an error on the coordinator. -CREATE DATABASE lc_collate_test LC_COLLATE = 'en_US.UTF-8'; -CREATE DATABASE lc_ctype_test LC_CTYPE = 'en_US.UTF-8'; -CREATE DATABASE locale_test LOCALE = 'en_US.UTF-8'; -CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'; +CREATE DATABASE lc_collate_test LC_COLLATE = 'C.UTF-8'; +CREATE DATABASE lc_ctype_test LC_CTYPE = 'C.UTF-8'; +CREATE DATABASE locale_test LOCALE = 'C.UTF-8'; +CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'C.UTF-8' LC_CTYPE = 'C.UTF-8'; -- Test LC / LOCALE settings that match the ones provided in template db. CREATE DATABASE lc_collate_test LC_COLLATE = 'C'; From 30e1a858d214f5d65ed7ff0ed410327d1421a433 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 11:41:58 +0300 Subject: [PATCH 129/180] Removes oid support --- src/backend/distributed/deparser/deparse_database_stmts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index bba6bedb4bc..3383d29540e 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -259,7 +259,7 @@ DeparseAlterDatabaseSetStmt(Node *node) static void ValidateCreateDatabaseOptions(DefElem *option) { - if (strcmp(option->defname, "strategy") == 0) + if (strcmp(option->defname, "strategy") == 0 || strcmp(option->defname, "oid") == 0) { ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", From 2ca6e2c3580db227f86743caac6dbd0eed3d164c Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 11:43:31 +0300 Subject: [PATCH 130/180] indent --- src/backend/distributed/metadata/metadata_sync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 0c04502e6dc..5aa01b8acc2 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3959,8 +3959,8 @@ citus_internal_database_command(PG_FUNCTION_ARGS) else { ereport(ERROR, (errmsg("citus_internal_database_command() can only be used " - "for CREATE DATABASE and DROP DATABASE commands by " - "Citus."))); + "for CREATE DATABASE and DROP DATABASE commands by " + "Citus."))); } /* rollback GUCs to the state before this session */ From 69217678b3ebd3e231531e752208078c877b6064 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 12:16:08 +0300 Subject: [PATCH 131/180] allow wal_log option only if it's set to default --- .../distributed/deparser/deparse_database_stmts.c | 13 ++++++++++--- src/include/distributed/commands.h | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 3383d29540e..2c1dc0b66e8 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -259,7 +259,7 @@ DeparseAlterDatabaseSetStmt(Node *node) static void ValidateCreateDatabaseOptions(DefElem *option) { - if (strcmp(option->defname, "strategy") == 0 || strcmp(option->defname, "oid") == 0) + if (strcmp(option->defname, "oid") == 0) { ereport(ERROR, errmsg("CREATE DATABASE option \"%s\" is not supported", @@ -267,11 +267,18 @@ ValidateCreateDatabaseOptions(DefElem *option) } char *optionValue = defGetString(option); + if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, "template1") != 0) { - ereport(ERROR, errmsg( - "Only template1 is supported as template parameter for CREATE DATABASE")); + ereport(ERROR, errmsg("Only template1 is supported as template " + "parameter for CREATE DATABASE")); } + + if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != 0) + { + ereport(ERROR, errmsg("Only wal_log is supported as strategy " + "parameter for CREATE DATABASE")); + } } diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 85c55d39a82..4c47d3ecd21 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -239,9 +239,9 @@ extern List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, extern List * PostprocessCreateDatabaseStmt(Node *node, const char *queryString); extern List * PreprocessDropDatabaseStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext); -extern List * DropDatabaseStmtObjectAddress(Node *node, bool missing_ok, +extern List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); -extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missing_ok, +extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * GenerateCreateDatabaseCommandList(void); From ffa1fa09637c576e19ad31df7623d0252c673f0c Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 12:38:43 +0300 Subject: [PATCH 132/180] Improve tests for >= pg15 & pg >= 16 --- .../create_drop_database_propagation.out | 150 ++++---- .../create_drop_database_propagation_pg15.out | 325 +++--------------- .../create_drop_database_propagation_pg16.out | 23 ++ ...reate_drop_database_propagation_pg16_0.out | 9 + .../regress/expected/multi_test_helpers.out | 30 +- src/test/regress/multi_1_schedule | 1 + .../create_drop_database_propagation_pg15.sql | 255 ++------------ .../create_drop_database_propagation_pg16.sql | 22 ++ src/test/regress/sql/multi_test_helpers.sql | 30 +- 9 files changed, 265 insertions(+), 580 deletions(-) create mode 100644 src/test/regress/expected/create_drop_database_propagation_pg16.out create mode 100644 src/test/regress/expected/create_drop_database_propagation_pg16_0.out create mode 100644 src/test/regress/sql/create_drop_database_propagation_pg16.sql diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 00f95c7c306..18bfa0c8ab5 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -68,9 +68,9 @@ DETAIL: Citus does not propagate CREATE DATABASE command to workers HINT: You can manually create a database and its extensions on workers. -- check that it's only created for coordinator SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) @@ -92,10 +92,10 @@ DETAIL: Citus does not propagate CREATE DATABASE command to workers HINT: You can manually create a database and its extensions on workers. -- check that it's only created for coordinator SELECT * FROM public.check_database_on_all_nodes('local_database') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- coordinator (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (local) | {"database_properties": {"datacl": null, "datname": "local_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) @@ -131,11 +131,11 @@ CREATE DATABASE mydatabase_1 ALLOW_CONNECTIONS = true IS_TEMPLATE = false; SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -- Test LC / LOCALE settings that don't match the ones provided in template db. @@ -158,35 +158,35 @@ CREATE DATABASE lc_ctype_test LC_CTYPE = 'C'; CREATE DATABASE locale_test LOCALE = 'C'; CREATE DATABASE lc_collate_lc_ctype_test LC_COLLATE = 'C' LC_CTYPE = 'C'; SELECT * FROM public.check_database_on_all_nodes('lc_collate_test') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('lc_ctype_test') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('locale_test') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "locale_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('lc_collate_lc_ctype_test') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "lc_collate_lc_ctype_test", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) DROP DATABASE lc_collate_test; @@ -223,11 +223,11 @@ SELECT result FROM run_command_on_all_nodes( (3 rows) SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -- test database syncing after node addition @@ -246,10 +246,10 @@ CREATE DATABASE mydatabase ALLOW_CONNECTIONS = false IS_TEMPLATE = false; SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (2 rows) SET citus.metadata_sync_mode to 'transactional'; @@ -260,27 +260,27 @@ select 1 from citus_add_node('localhost', :worker_2_port); (1 row) SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) select 1 from citus_remove_node('localhost', :worker_2_port); @@ -298,27 +298,27 @@ select 1 from citus_add_node('localhost', :worker_2_port); RESET citus.metadata_sync_mode; SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT citus_disable_node_and_wait('localhost', :worker_1_port, true); @@ -335,35 +335,35 @@ SELECT 1 FROM citus_activate_node('localhost', :worker_1_port); (1 row) SELECT * FROM public.check_database_on_all_nodes('mydatabase') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('mydatabase_1') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "mydatabase_1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": 10, "daticulocale": null, "datistemplate": false, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('db_needs_escape') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db_needs_escape", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "role-needs\\!escape", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SELECT * FROM public.check_database_on_all_nodes('test_node_activation') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_node_activation", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) SET citus.log_remote_commands = true; @@ -400,11 +400,11 @@ CREATE DATABASE my_template_database ALLOW_CONNECTIONS = false IS_TEMPLATE = true; SELECT * FROM public.check_database_on_all_nodes('my_template_database') ORDER BY node_type; - node_type | result + node_type | result --------------------------------------------------------------------- - coordinator (local) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} - worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + coordinator (local) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": true, "database_owner": "create_drop_db_test_user", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) --template databases could not be dropped so we need to change the template flag diff --git a/src/test/regress/expected/create_drop_database_propagation_pg15.out b/src/test/regress/expected/create_drop_database_propagation_pg15.out index bc637480350..9a501558a2b 100644 --- a/src/test/regress/expected/create_drop_database_propagation_pg15.out +++ b/src/test/regress/expected/create_drop_database_propagation_pg15.out @@ -8,298 +8,75 @@ SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 \else \q \endif --- create/drop database for pg > 15 -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; -\c - - - :master_port -create user create_drop_db_test_user; +-- create/drop database for pg >= 15 set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; CREATE DATABASE mydatabase - WITH - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE DATABASE mydatabase OWNER create_drop_db_test_user CONNECTION LIMIT 10 ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result + WITH OID = 966345; +ERROR: CREATE DATABASE option "oid" is not supported +CREATE DATABASE mydatabase + WITH strategy file_copy; +ERROR: Only wal_log is supported as strategy parameter for CREATE DATABASE +CREATE DATABASE st_wal_log + WITH strategy WaL_LoG; +SELECT * FROM public.check_database_on_all_nodes('st_wal_log') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": 10, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + coordinator (local) | {"database_properties": {"datacl": null, "datname": "st_wal_log", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "st_wal_log", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "st_wal_log", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase; -NOTICE: issuing DROP DATABASE mydatabase -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing DROP DATABASE mydatabase -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - - - -(3 rows) - -select citus_remove_node('localhost', :worker_2_port); - citus_remove_node +drop database st_wal_log; +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? --------------------------------------------------------------------- - + 1 (1 row) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -CREATE DATABASE mydatabase2 - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; -NOTICE: issuing CREATE DATABASE mydatabase2 OWNER create_drop_db_test_user ENCODING 'UTF8' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS true IS_TEMPLATE false OID 966345 -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] -(2 rows) - +-- test COLLATION_VERSION +CREATE DATABASE test_collation_version + WITH ENCODING = 'UTF8' + COLLATION_VERSION = '1.0' + ALLOW_CONNECTIONS = false; select 1 from citus_add_node('localhost', :worker_2_port); ?column? --------------------------------------------------------------------- 1 (1 row) -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - result +SELECT * FROM public.check_database_on_all_nodes('test_collation_version') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "mydatabase2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": true, "datconnlimit": -1, "datistemplate": false, "database_owner": "create_drop_db_test_user"}] + coordinator (local) | {"database_properties": {"datacl": null, "datname": "test_collation_version", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": "1.0", "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_collation_version", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": "1.0", "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_collation_version", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": false, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": "1.0", "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase2; -NOTICE: issuing DROP DATABASE mydatabase2 -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing DROP DATABASE mydatabase2 -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - result +drop database test_collation_version; +SET client_min_messages TO WARNING; +-- test LOCALE_PROVIDER & ICU_LOCALE +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'icu' + ICU_LOCALE = 'en_US'; +ERROR: new locale provider (icu) does not match locale provider of the template database (libc) +HINT: Use the same locale provider as in the template database, or use template0 as template. +RESET client_min_messages; +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'libc' + ICU_LOCALE = 'en_US'; +ERROR: ICU locale cannot be specified unless locale provider is ICU +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'libc'; +SELECT * FROM public.check_database_on_all_nodes('test_locale_provider') ORDER BY node_type; + node_type | result --------------------------------------------------------------------- - - - + coordinator (local) | {"database_properties": {"datacl": null, "datname": "test_locale_provider", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_locale_provider", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "test_locale_provider", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; --- create a template database with all options set and allow connections false -CREATE DATABASE my_template_database - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = false - IS_TEMPLATE = true; -NOTICE: issuing CREATE DATABASE my_template_database OWNER create_drop_db_test_user ENCODING 'UTF8' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE DATABASE my_template_database OWNER create_drop_db_test_user ENCODING 'UTF8' COLLATION_VERSION '1.0' TABLESPACE create_drop_db_tablespace ALLOW_CONNECTIONS false IS_TEMPLATE true -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] - [{"datacl": null, "datname": "my_template_database", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "create_drop_db_tablespace", "datallowconn": false, "datconnlimit": -1, "datistemplate": true, "database_owner": "create_drop_db_test_user"}] -(3 rows) - -SET citus.log_remote_commands = true; ---template databases could not be dropped so we need to change the template flag -SELECT result from run_command_on_all_nodes( - $$ - UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - UPDATE 1 - UPDATE 1 - UPDATE 1 -(3 rows) - -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; -NOTICE: issuing DROP DATABASE my_template_database -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing DROP DATABASE my_template_database -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - result ---------------------------------------------------------------------- - - - -(3 rows) - ---tests for special characters in database name -set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -create database "mydatabase#1'2"; -NOTICE: issuing CREATE DATABASE "mydatabase#1'2" -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE DATABASE "mydatabase#1'2" -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database if exists "mydatabase#1'2"; -NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing DROP DATABASE IF EXISTS "mydatabase#1'2" -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -\c - - - :master_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; +drop database test_locale_provider; \c - - - :master_port -drop user create_drop_db_test_user; diff --git a/src/test/regress/expected/create_drop_database_propagation_pg16.out b/src/test/regress/expected/create_drop_database_propagation_pg16.out new file mode 100644 index 00000000000..75cd99e616c --- /dev/null +++ b/src/test/regress/expected/create_drop_database_propagation_pg16.out @@ -0,0 +1,23 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q +\endif +-- create/drop database for pg >= 16 +set citus.enable_create_database_propagation=on; +-- test icu_rules +-- +-- practically we don't support it but better to test +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook'; +ERROR: ICU rules cannot be specified unless locale provider is ICU +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook' locale_provider='icu'; +ERROR: LOCALE or ICU_LOCALE must be specified +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook' locale_provider='icu' icu_locale = 'de_DE'; +NOTICE: using standard form "de-DE" for ICU locale "de_DE" +ERROR: new locale provider (icu) does not match locale provider of the template database (libc) +HINT: Use the same locale provider as in the template database, or use template0 as template. diff --git a/src/test/regress/expected/create_drop_database_propagation_pg16_0.out b/src/test/regress/expected/create_drop_database_propagation_pg16_0.out new file mode 100644 index 00000000000..730c916cadc --- /dev/null +++ b/src/test/regress/expected/create_drop_database_propagation_pg16_0.out @@ -0,0 +1,9 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index da771b4c5eb..1d077bb1170 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -534,7 +534,22 @@ $func$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) RETURNS TABLE (node_type text, result text) AS $func$ +DECLARE + pg_ge_15_options text := ''; + pg_ge_16_options text := ''; BEGIN + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'datlocprovider') THEN + pg_ge_15_options := ', daticulocale, datcollversion, datlocprovider'; + ELSE + pg_ge_15_options := $$, null as daticulocale, null as datcollversion, 'c' as datlocprovider$$; + END IF; + + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'daticurules') THEN + pg_ge_16_options := ', daticurules'; + ELSE + pg_ge_16_options := ', null as daticurules'; + END IF; + RETURN QUERY SELECT CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' @@ -553,19 +568,20 @@ BEGIN SELECT to_jsonb(database_properties.*) FROM ( SELECT datname, pa.rolname as database_owner, - pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, - datistemplate, datallowconn, datconnlimit, - pt.spcname AS tablespace, datcollate, datctype, daticulocale, - datcollversion, datacl + pg_encoding_to_char(pd.encoding) as encoding, + datistemplate, datallowconn, datconnlimit, datacl, + pt.spcname AS tablespace, datcollate, datctype + %2$s -- >= pg15 options + %3$s -- >= pg16 options FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid JOIN pg_tablespace pt ON pd.dattablespace = pt.oid - WHERE datname = '%s' + WHERE datname = '%1$s' ) database_properties ) AS database_properties, ( SELECT COUNT(*)=1 - FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%1$s') ) AS pg_dist_object_record_for_db_exists, ( SELECT COUNT(*) > 0 @@ -574,7 +590,7 @@ BEGIN ) AS stale_pg_dist_object_record_for_a_db_exists ) q $$, - p_database_name, p_database_name + p_database_name, pg_ge_15_options, pg_ge_16_options ) ) q2 JOIN pg_dist_node USING (nodeid); diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 73696bde697..c376d44a82c 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -36,6 +36,7 @@ test: distributed_triggers test: create_single_shard_table test: create_drop_database_propagation test: create_drop_database_propagation_pg15 +test: create_drop_database_propagation_pg16 # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential test: single_shard_table_udfs diff --git a/src/test/regress/sql/create_drop_database_propagation_pg15.sql b/src/test/regress/sql/create_drop_database_propagation_pg15.sql index ca3e3b20235..40d1b9e0987 100644 --- a/src/test/regress/sql/create_drop_database_propagation_pg15.sql +++ b/src/test/regress/sql/create_drop_database_propagation_pg15.sql @@ -9,236 +9,57 @@ SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15 \q \endif --- create/drop database for pg > 15 +-- create/drop database for pg >= 15 +set citus.enable_create_database_propagation=on; -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts3' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +CREATE DATABASE mydatabase + WITH OID = 966345; -\c - - - :worker_1_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts4' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +CREATE DATABASE mydatabase + WITH strategy file_copy; -\c - - - :worker_2_port -\set create_drop_db_tablespace :abs_srcdir '/tmp_check/ts5' -CREATE TABLESPACE create_drop_db_tablespace LOCATION :'create_drop_db_tablespace'; +CREATE DATABASE st_wal_log + WITH strategy WaL_LoG; -\c - - - :master_port -create user create_drop_db_test_user; -set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; -CREATE DATABASE mydatabase - WITH - OWNER = create_drop_db_test_user - CONNECTION LIMIT = 10 - ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase; - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - -select citus_remove_node('localhost', :worker_2_port); - - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; - -CREATE DATABASE mydatabase2 - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = true - IS_TEMPLATE = false - OID = 966345; - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; +SELECT * FROM public.check_database_on_all_nodes('st_wal_log') ORDER BY node_type; +drop database st_wal_log; -select 1 from citus_add_node('localhost', :worker_2_port); +select 1 from citus_remove_node('localhost', :worker_2_port); -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase2' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database mydatabase2; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'mydatabase' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; - --- create a template database with all options set and allow connections false -CREATE DATABASE my_template_database - WITH OWNER = create_drop_db_test_user - ENCODING = 'UTF8' +-- test COLLATION_VERSION + +CREATE DATABASE test_collation_version + WITH ENCODING = 'UTF8' COLLATION_VERSION = '1.0' - TABLESPACE = create_drop_db_tablespace - ALLOW_CONNECTIONS = false - IS_TEMPLATE = true; - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - - -SET citus.log_remote_commands = false; - -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - -SET citus.log_remote_commands = true; - ---template databases could not be dropped so we need to change the template flag -SELECT result from run_command_on_all_nodes( - $$ - UPDATE pg_database SET datistemplate = false WHERE datname = 'my_template_database' - $$ -) ORDER BY result; - - -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database my_template_database; - -SET citus.log_remote_commands = false; -SELECT result from run_command_on_all_nodes( - $$ - SELECT jsonb_agg(to_jsonb(q2.*)) FROM ( - SELECT pd.datname, pg_encoding_to_char(pd.encoding) as encoding, - pd.datistemplate, pd.datallowconn, pd.datconnlimit, - pd.datcollate , pd. datctype , pd.datacl, - pa.rolname AS database_owner, pt.spcname AS tablespace - FROM pg_database pd - JOIN pg_authid pa ON pd.datdba = pa.oid - join pg_tablespace pt on pd.dattablespace = pt.oid - WHERE datname = 'my_template_database' - ) q2 - $$ -) ORDER BY result; - - ---tests for special characters in database name -set citus.enable_create_database_propagation=on; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%CREATE DATABASE%'; + ALLOW_CONNECTIONS = false; -create database "mydatabase#1'2"; +select 1 from citus_add_node('localhost', :worker_2_port); -set citus.grep_remote_commands = '%DROP DATABASE%'; -drop database if exists "mydatabase#1'2"; +SELECT * FROM public.check_database_on_all_nodes('test_collation_version') ORDER BY node_type; -\c - - - :master_port -drop tablespace create_drop_db_tablespace; +drop database test_collation_version; + +SET client_min_messages TO WARNING; +-- test LOCALE_PROVIDER & ICU_LOCALE +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'icu' + ICU_LOCALE = 'en_US'; +RESET client_min_messages; + +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'libc' + ICU_LOCALE = 'en_US'; + +CREATE DATABASE test_locale_provider + WITH ENCODING = 'UTF8' + LOCALE_PROVIDER = 'libc'; -\c - - - :worker_1_port -drop tablespace create_drop_db_tablespace; +SELECT * FROM public.check_database_on_all_nodes('test_locale_provider') ORDER BY node_type; -\c - - - :worker_2_port -drop tablespace create_drop_db_tablespace; +drop database test_locale_provider; \c - - - :master_port -drop user create_drop_db_test_user; diff --git a/src/test/regress/sql/create_drop_database_propagation_pg16.sql b/src/test/regress/sql/create_drop_database_propagation_pg16.sql new file mode 100644 index 00000000000..cec55381325 --- /dev/null +++ b/src/test/regress/sql/create_drop_database_propagation_pg16.sql @@ -0,0 +1,22 @@ +-- +-- PG16 +-- +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int >= 16 AS server_version_ge_16 +\gset +\if :server_version_ge_16 +\else +\q +\endif + +-- create/drop database for pg >= 16 + +set citus.enable_create_database_propagation=on; + +-- test icu_rules +-- +-- practically we don't support it but better to test + +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook'; +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook' locale_provider='icu'; +CREATE DATABASE citus_icu_rules_test WITH icu_rules='de_DE@collation=phonebook' locale_provider='icu' icu_locale = 'de_DE'; diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 2e8654ce201..835174169e3 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -560,7 +560,22 @@ $func$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) RETURNS TABLE (node_type text, result text) AS $func$ +DECLARE + pg_ge_15_options text := ''; + pg_ge_16_options text := ''; BEGIN + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'datlocprovider') THEN + pg_ge_15_options := ', daticulocale, datcollversion, datlocprovider'; + ELSE + pg_ge_15_options := $$, null as daticulocale, null as datcollversion, 'c' as datlocprovider$$; + END IF; + + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'daticurules') THEN + pg_ge_16_options := ', daticurules'; + ELSE + pg_ge_16_options := ', null as daticurules'; + END IF; + RETURN QUERY SELECT CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' @@ -579,19 +594,20 @@ BEGIN SELECT to_jsonb(database_properties.*) FROM ( SELECT datname, pa.rolname as database_owner, - pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, - datistemplate, datallowconn, datconnlimit, - pt.spcname AS tablespace, datcollate, datctype, daticulocale, - datcollversion, datacl + pg_encoding_to_char(pd.encoding) as encoding, + datistemplate, datallowconn, datconnlimit, datacl, + pt.spcname AS tablespace, datcollate, datctype + %2$s -- >= pg15 options + %3$s -- >= pg16 options FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid JOIN pg_tablespace pt ON pd.dattablespace = pt.oid - WHERE datname = '%s' + WHERE datname = '%1$s' ) database_properties ) AS database_properties, ( SELECT COUNT(*)=1 - FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%1$s') ) AS pg_dist_object_record_for_db_exists, ( SELECT COUNT(*) > 0 @@ -600,7 +616,7 @@ BEGIN ) AS stale_pg_dist_object_record_for_a_db_exists ) q $$, - p_database_name, p_database_name + p_database_name, pg_ge_15_options, pg_ge_16_options ) ) q2 JOIN pg_dist_node USING (nodeid); From 1a80181a9c125e9c6a31d26098e44aaf17af9d4d Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 13:42:11 +0300 Subject: [PATCH 133/180] indent --- .../distributed/deparser/deparse_database_stmts.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 2c1dc0b66e8..4551ccd1de7 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -271,14 +271,14 @@ ValidateCreateDatabaseOptions(DefElem *option) if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, "template1") != 0) { ereport(ERROR, errmsg("Only template1 is supported as template " - "parameter for CREATE DATABASE")); + "parameter for CREATE DATABASE")); } - if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != 0) - { - ereport(ERROR, errmsg("Only wal_log is supported as strategy " - "parameter for CREATE DATABASE")); - } + if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != 0) + { + ereport(ERROR, errmsg("Only wal_log is supported as strategy " + "parameter for CREATE DATABASE")); + } } From 8728871cc313f29162892b84caec24c9a94fbe4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 14:56:24 +0300 Subject: [PATCH 134/180] Apply suggestions from code review Co-authored-by: Onur Tirtir --- src/backend/distributed/commands/database.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 2d0a2ce1692..07cfda8056c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -296,7 +296,7 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, /* - * PostprocessCreatedbStmt is executed after the statement is applied to the local + * PostprocessCreateDatabaseStmt is executed after the statement is applied to the local * postgres instance. In this stage we can prepare the commands that need to be run on * all workers to create the database. Since the CREATE DATABASE statement gives error * in a transaction block, we need to use NontransactionalNodeDDLTaskList to send the @@ -324,7 +324,7 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) /* - * PostprocessAlterDatabaseStmt is executed after the statement is applied to the local + * PreprocessDropDatabaseStmt is executed after the statement is applied to the local * postgres instance. In this stage we can prepare the commands that need to be run on * all workers to drop the database. Since the DROP DATABASE statement gives error in * transaction context, we need to use NontransactionalNodeDDLTaskList to send the From 52c9e92544341758b1ada76ec311303bc3c096c2 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 13:44:08 +0300 Subject: [PATCH 135/180] Adds IsAnyObjectDistributed check for db --- src/backend/distributed/metadata/distobject.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 94c12d47f5d..9c5b6bf6162 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -386,7 +386,10 @@ UnmarkNodeWideObjectsDistributed(Node *node) Oid dbOid = get_database_oid(dbName, stmt->missing_ok); ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, dbOid); - UnmarkObjectDistributed(dbObjectAddress); + if(IsAnyObjectDistributed(list_make1(dbObjectAddress))) + { + UnmarkObjectDistributed(dbObjectAddress); + } } } From 3731c45c295c7ec862d9ef4e7eb6b96d485c5e35 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 14:19:19 +0300 Subject: [PATCH 136/180] Fixes drop force option --- .../deparser/deparse_database_stmts.c | 20 ++++++++++++++++++- .../create_drop_database_propagation.out | 18 +++++++++++++++++ .../sql/create_drop_database_propagation.sql | 14 +++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 4551ccd1de7..32b89b419bd 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -329,11 +329,22 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) DefElem *option = NULL; + foreach_ptr(option, stmt->options) { + //if it is the first option then append with "WITH" else append with "," + if (option == linitial(stmt->options)) + { + appendStringInfo(buf, " WITH ( "); + } + else + { + appendStringInfo(buf, ", "); + } + if (strcmp(option->defname, "force") == 0) { - appendStringInfo(buf, " FORCE"); + appendStringInfo(buf, "FORCE"); } else { @@ -341,6 +352,13 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) errmsg("unrecognized DROP DATABASE option \"%s\"", option->defname))); } + + //if it is the last option then append with ")" + if (option == llast(stmt->options)) + { + appendStringInfo(buf, " )"); + } + } } diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 18bfa0c8ab5..7bd0cc8fa40 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -477,6 +477,24 @@ SET citus.enable_create_database_propagation TO ON; DROP DATABASE test_node_activation; DROP DATABASE db_needs_escape; DROP USER "role-needs\!escape"; +-- drop database with force options test +create database db_force_test; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; +drop database db_force_test with (force); +NOTICE: issuing DROP DATABASE db_force_test WITH ( FORCE ) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing DROP DATABASE db_force_test WITH ( FORCE ) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +reset citus.log_remote_commands; +SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 6e6ecacb5ba..1b2f96da2ef 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -261,6 +261,20 @@ DROP DATABASE test_node_activation; DROP DATABASE db_needs_escape; DROP USER "role-needs\!escape"; +-- drop database with force options test + +create database db_force_test; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%DROP DATABASE%'; + +drop database db_force_test with (force); + +reset citus.log_remote_commands; + +SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type; + + --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. From 712fd8ebf358a341b99ce44c58805c1377d6b1d3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 14:23:37 +0300 Subject: [PATCH 137/180] Fixes comments --- src/backend/distributed/deparser/deparse_database_stmts.c | 5 ++--- src/backend/distributed/metadata/distobject.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 32b89b419bd..34c4a2dce85 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -332,7 +332,7 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) foreach_ptr(option, stmt->options) { - //if it is the first option then append with "WITH" else append with "," + /*if it is the first option then append with "WITH" else append with "," */ if (option == linitial(stmt->options)) { appendStringInfo(buf, " WITH ( "); @@ -353,12 +353,11 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) option->defname))); } - //if it is the last option then append with ")" + /*if it is the last option then append with ")" */ if (option == llast(stmt->options)) { appendStringInfo(buf, " )"); } - } } diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 9c5b6bf6162..9f31add60b8 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -386,7 +386,7 @@ UnmarkNodeWideObjectsDistributed(Node *node) Oid dbOid = get_database_oid(dbName, stmt->missing_ok); ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, dbOid); - if(IsAnyObjectDistributed(list_make1(dbObjectAddress))) + if (IsAnyObjectDistributed(list_make1(dbObjectAddress))) { UnmarkObjectDistributed(dbObjectAddress); } From a9977e884018ba0380185c4aa95513d4ca11369c Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 15:49:19 +0300 Subject: [PATCH 138/180] Moves validation to preprocess for createdb --- src/backend/distributed/commands/database.c | 55 ++++++++++++++++++- .../deparser/deparse_database_stmts.c | 33 +---------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 07cfda8056c..82b7eed972f 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -21,6 +21,7 @@ #include "catalog/pg_database_d.h" #include "catalog/pg_tablespace.h" #include "commands/dbcommands.h" +#include "commands/defrem.h" #include "nodes/parsenodes.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -270,6 +271,55 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, } +/* + * This function validates the options provided for the CREATE DATABASE command. + * It iterates over each option in the stmt->options list and checks if it's supported. + * If an unsupported option is found, or if a supported option has an invalid value, + * it raises an error. + * + * Parameters: + * stmt: A CreatedbStmt struct representing a CREATE DATABASE command. + * The options field is a list of DefElem structs, each representing an option. + * + * Currently, this function checks for the following: + * - The "oid" option is not supported. + * - The "template" option is only supported with the value "template1". + * - The "strategy" option is only supported with the value "wal_log". + * + * If any of these checks fail, the function calls ereport to raise an error. + */ +static void +EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt) +{ + DefElem *option = NULL; + foreach_ptr(option, stmt->options) + { + if (strcmp(option->defname, "oid") == 0) + { + ereport(ERROR, + errmsg("CREATE DATABASE option \"%s\" is not supported", + option->defname)); + } + + char *optionValue = defGetString(option); + + if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, + "template1") != 0) + { + ereport(ERROR, errmsg("Only template1 is supported as template " + "parameter for CREATE DATABASE")); + } + + if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != + 0) + { + ereport(ERROR, errmsg("Only wal_log is supported as strategy " + "parameter for CREATE DATABASE")); + } + } +} + + /* * PostprocessAlterDatabaseStmt is executed before the statement is applied to the local * postgres instance. @@ -288,8 +338,9 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - /*Validate the statement */ - DeparseTreeNode(node); + /*validate the statement*/ + CreatedbStmt *stmt = castNode(CreatedbStmt, node); + EnsureSupportedCreateDatabaseCommand(stmt); return NIL; } diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 34c4a2dce85..e726fa84529 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -251,37 +251,6 @@ DeparseAlterDatabaseSetStmt(Node *node) } -/* - * Validates for if option is template, lc_type, locale or lc_collate, propagation will - * not be supported since template and strategy options are not stored in the catalog - * and lc_type, locale and lc_collate options depends on template parameter. - */ -static void -ValidateCreateDatabaseOptions(DefElem *option) -{ - if (strcmp(option->defname, "oid") == 0) - { - ereport(ERROR, - errmsg("CREATE DATABASE option \"%s\" is not supported", - option->defname)); - } - - char *optionValue = defGetString(option); - - if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, "template1") != 0) - { - ereport(ERROR, errmsg("Only template1 is supported as template " - "parameter for CREATE DATABASE")); - } - - if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != 0) - { - ereport(ERROR, errmsg("Only wal_log is supported as strategy " - "parameter for CREATE DATABASE")); - } -} - - static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { @@ -293,7 +262,7 @@ AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) foreach_ptr(option, stmt->options) { - ValidateCreateDatabaseOptions(option); + /*ValidateCreateDatabaseOptions(option); */ DefElemOptionToStatement(buf, option, create_database_option_formats, lengthof(create_database_option_formats)); From fcdea98edd228580e15c9fa2e56885fd178b7844 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 15:58:09 +0300 Subject: [PATCH 139/180] Removes drop in citus_internal_db_command udf --- .../distributed/metadata/metadata_sync.c | 18 +++--------------- .../create_drop_database_propagation.out | 4 ++-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 5aa01b8acc2..56a0a8566b7 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3895,7 +3895,7 @@ citus_internal_update_none_dist_table_metadata(PG_FUNCTION_ARGS) /* * citus_internal_database_command is an internal UDF to - * create/drop a database in an idempotent maner without + * create a database in an idempotent maner without * transaction block restrictions. */ Datum @@ -3925,7 +3925,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) GUC_ACTION_LOCAL, true, 0, false); /* - * createdb() / DropDatabase() uses ParseState to report the error position for the + * createdb() uses ParseState to report the error position for the * input command and the position is reported to be 0 when it's provided as NULL. * We're okay with that because we don't expect this UDF to be called with an incorrect * DDL command. @@ -3944,22 +3944,10 @@ citus_internal_database_command(PG_FUNCTION_ARGS) createdb(pstate, (CreatedbStmt *) parseTree); } } - else if (IsA(parseTree, DropdbStmt)) - { - DropdbStmt *stmt = castNode(DropdbStmt, parseTree); - - bool missingOk = false; - Oid databaseOid = get_database_oid(stmt->dbname, missingOk); - - if (OidIsValid(databaseOid)) - { - DropDatabase(pstate, (DropdbStmt *) parseTree); - } - } else { ereport(ERROR, (errmsg("citus_internal_database_command() can only be used " - "for CREATE DATABASE and DROP DATABASE commands by " + "for CREATE DATABASE command by " "Citus."))); } diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 7bd0cc8fa40..5ab27ba34a4 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -25,9 +25,9 @@ SELECT pg_catalog.citus_internal_database_command(null); ERROR: command cannot be NULL -- fails on non create / drop db command SELECT pg_catalog.citus_internal_database_command('CREATE TABLE foo_bar(a int)'); -ERROR: citus_internal_database_command() can only be used for CREATE DATABASE and DROP DATABASE commands by Citus. +ERROR: citus_internal_database_command() can only be used for CREATE DATABASE command by Citus. SELECT pg_catalog.citus_internal_database_command('SELECT 1'); -ERROR: citus_internal_database_command() can only be used for CREATE DATABASE and DROP DATABASE commands by Citus. +ERROR: citus_internal_database_command() can only be used for CREATE DATABASE command by Citus. SELECT pg_catalog.citus_internal_database_command('asfsfdsg'); ERROR: syntax error at or near "asfsfdsg" SELECT pg_catalog.citus_internal_database_command(''); From b5388ddf929517dacc2c4d5c14986459584675ab Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 13 Nov 2023 17:13:56 +0300 Subject: [PATCH 140/180] Adds icu_rules in meta_data sync --- src/backend/distributed/commands/database.c | 30 +++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 82b7eed972f..cf6ee36a5d0 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -52,8 +52,9 @@ typedef struct DatabaseCollationInfo char *collation; char *ctype; #if PG_VERSION_NUM >= PG_VERSION_15 - char *icu_locale; + char *icuLocale; char *collversion; + char *icuRules; #endif } DatabaseCollationInfo; @@ -529,11 +530,11 @@ GetDatabaseCollation(Oid dbOid) &isNull); if (isNull) { - info.icu_locale = NULL; + info.icuLocale = NULL; } else { - info.icu_locale = TextDatumGetCString(icuLocaleDatum); + info.icuLocale = TextDatumGetCString(icuLocaleDatum); } Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, @@ -548,6 +549,20 @@ GetDatabaseCollation(Oid dbOid) } #endif + #if PG_VERSION_NUM >= PG_VERSION_16 + Datum icuRulesDatum = heap_getattr(tup, Anum_pg_database_daticurules, tupdesc, + &isNull); + + if (isNull) + { + info.icuRules = NULL; + } + else + { + info.icuRules = TextDatumGetCString(icuRulesDatum); + } + #endif + table_close(rel, AccessShareLock); UnregisterSnapshot(snapshot); heap_freetuple(tup); @@ -630,10 +645,10 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) } #if PG_VERSION_NUM >= PG_VERSION_15 - if (collInfo.icu_locale != NULL) + if (collInfo.icuLocale != NULL) { appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr( - collInfo.icu_locale)); + collInfo.icuLocale)); } if (databaseForm->datlocprovider != 0) @@ -665,6 +680,11 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); } + if(collInfo.icuRules != NULL){ + appendStringInfo(&str, " ICU_RULES = %s", + quote_literal_cstr(collInfo.icuRules)); + } + appendStringInfo(&str, " IS_TEMPLATE = %s", quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); From 7e15939e500f2f2384b9a4593494169109150113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 17:15:11 +0300 Subject: [PATCH 141/180] Update src/backend/distributed/commands/database.c Co-authored-by: Onur Tirtir --- src/backend/distributed/commands/database.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 82b7eed972f..94477a47b84 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -594,6 +594,10 @@ GetLocaleProviderString(char datlocprovider) /* * GenerateCreateDatabaseStatementFromPgDatabase gets the pg_database tuple and returns the * CREATE DATABASE statement that can be used to create given database. + * + * Note that this doesn't deparse OID of the database and this is not a + * problem as we anyway don't allow specifying custom OIDs for databases + * when creating them. */ static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) From c1dce6fc2bfc2b359b00cd88a14333d24b9c5c2e Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 13 Nov 2023 18:00:42 +0300 Subject: [PATCH 142/180] commit --- src/backend/distributed/commands/database.c | 158 +++++++++----------- 1 file changed, 67 insertions(+), 91 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 94477a47b84..0d00bb6018c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -49,12 +49,17 @@ */ typedef struct DatabaseCollationInfo { - char *collation; - char *ctype; - #if PG_VERSION_NUM >= PG_VERSION_15 - char *icu_locale; - char *collversion; - #endif + char *datcollate; + char *datctype; + +#if PG_VERSION_NUM >= PG_VERSION_15 + char *daticulocale; + char *datcollversion; +#endif + +#if PG_VERSION_NUM >= PG_VERSION_16 + char *daticurules; +#endif } DatabaseCollationInfo; static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); @@ -485,15 +490,14 @@ GetTablespaceName(Oid tablespaceOid) /* * GetDatabaseCollation gets oid of a database and returns all the collation related information - * We need this method since collation related info in Form_pg_database is not accessible + * We need this method since collation related info in Form_pg_database is not accessible. */ static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid) { DatabaseCollationInfo info; - bool isNull; + memset(&info, 0, sizeof(DatabaseCollationInfo)); - Snapshot snapshot = RegisterSnapshot(GetLatestSnapshot()); Relation rel = table_open(DatabaseRelationId, AccessShareLock); HeapTuple tup = get_catalog_object_by_oid(rel, Anum_pg_database_oid, dbOid); if (!HeapTupleIsValid(tup)) @@ -501,55 +505,44 @@ GetDatabaseCollation(Oid dbOid) elog(ERROR, "cache lookup failed for database %u", dbOid); } + bool isNull = false; + TupleDesc tupdesc = RelationGetDescr(rel); + Datum collationDatum = heap_getattr(tup, Anum_pg_database_datcollate, tupdesc, &isNull); - if (isNull) - { - info.collation = NULL; - } - else - { - info.collation = TextDatumGetCString(collationDatum); - } + info.datcollate = TextDatumGetCString(collationDatum); Datum ctypeDatum = heap_getattr(tup, Anum_pg_database_datctype, tupdesc, &isNull); - if (isNull) - { - info.ctype = NULL; - } - else - { - info.ctype = TextDatumGetCString(ctypeDatum); - } + info.datctype = TextDatumGetCString(ctypeDatum); - #if PG_VERSION_NUM >= PG_VERSION_15 +#if PG_VERSION_NUM >= PG_VERSION_15 Datum icuLocaleDatum = heap_getattr(tup, Anum_pg_database_daticulocale, tupdesc, &isNull); - if (isNull) - { - info.icu_locale = NULL; - } - else + if (!isNull) { - info.icu_locale = TextDatumGetCString(icuLocaleDatum); + info.daticulocale = TextDatumGetCString(icuLocaleDatum); } Datum collverDatum = heap_getattr(tup, Anum_pg_database_datcollversion, tupdesc, &isNull); - if (isNull) + if (!isNull) { - info.collversion = NULL; + info.datcollversion = TextDatumGetCString(collverDatum); } - else +#endif + +#if PG_VERSION_NUM >= PG_VERSION_16 + Datum icurulesDatum = heap_getattr(tup, Anum_pg_database_daticurules, tupdesc, + &isNull); + if (!isNull) { - info.collversion = TextDatumGetCString(collverDatum); + info.daticurules = TextDatumGetCString(icurulesDatum); } - #endif +#endif table_close(rel, AccessShareLock); - UnregisterSnapshot(snapshot); heap_freetuple(tup); return info; @@ -577,13 +570,11 @@ GetLocaleProviderString(char datlocprovider) return "icu"; } - case 'l': + default: { - return "locale"; + ereport(ERROR, (errmsg("unexpected datlocprovider value: %c", + datlocprovider))); } - - default: - return ""; } } @@ -610,68 +601,53 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, "CREATE DATABASE %s", quote_identifier(NameStr(databaseForm->datname))); - if (databaseForm->datdba != InvalidOid) - { - appendStringInfo(&str, " OWNER = %s", - quote_literal_cstr(GetUserNameFromId(databaseForm->datdba, - false))); - } + appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); - if (databaseForm->encoding != -1) - { - appendStringInfo(&str, " ENCODING = %s", - quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); - } + appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", + quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); - if (collInfo.collation != NULL) - { - appendStringInfo(&str, " LC_COLLATE = %s", quote_literal_cstr( - collInfo.collation)); - } - if (collInfo.ctype != NULL) - { - appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.ctype)); - } + appendStringInfo(&str, " IS_TEMPLATE = %s", + quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); - #if PG_VERSION_NUM >= PG_VERSION_15 - if (collInfo.icu_locale != NULL) - { - appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr( - collInfo.icu_locale)); - } + appendStringInfo(&str, " LC_COLLATE = %s", + quote_literal_cstr(collInfo.datcollate)); - if (databaseForm->datlocprovider != 0) - { - appendStringInfo(&str, " LOCALE_PROVIDER = %s", - quote_literal_cstr(GetLocaleProviderString( - databaseForm->datlocprovider))); - } + appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.datctype)); - if (collInfo.collversion != NULL) + appendStringInfo(&str, " OWNER = %s", + quote_literal_cstr(GetUserNameFromId(databaseForm->datdba, false))); + + appendStringInfo(&str, " TABLESPACE = %s", + quote_identifier(GetTablespaceName(databaseForm->dattablespace))); + + appendStringInfo(&str, " ENCODING = %s", + quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); + +#if PG_VERSION_NUM >= PG_VERSION_15 + if (collInfo.datcollversion != NULL) { - appendStringInfo(&str, " COLLATION_VERSION = %s", quote_literal_cstr( - collInfo.collversion)); + appendStringInfo(&str, " COLLATION_VERSION = %s", + quote_literal_cstr(collInfo.datcollversion)); } - #endif - if (databaseForm->dattablespace != InvalidOid) + if (collInfo.daticulocale != NULL) { - appendStringInfo(&str, " TABLESPACE = %s", - quote_identifier(GetTablespaceName( - databaseForm->dattablespace))); + appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr( + collInfo.daticulocale)); } - appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", - quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); + appendStringInfo(&str, " LOCALE_PROVIDER = %s", + quote_literal_cstr(GetLocaleProviderString( + databaseForm->datlocprovider))); +#endif - if (databaseForm->datconnlimit >= 0) +#if PG_VERSION_NUM >= PG_VERSION_16 + if (collInfo.daticurules != NULL) { - appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); + appendStringInfo(&str, " ICU_RULES = %s", quote_literal_cstr( + collInfo.daticurules)); } - - appendStringInfo(&str, " IS_TEMPLATE = %s", - quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); - +#endif return str.data; } From 32df3313d449c70d18944bc2cbfe650d5a4d0653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 18:59:36 +0300 Subject: [PATCH 143/180] Update src/backend/distributed/metadata/metadata_sync.c Co-authored-by: Onur Tirtir --- src/backend/distributed/metadata/metadata_sync.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 56a0a8566b7..cb74ebcb5c1 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -3947,8 +3947,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) else { ereport(ERROR, (errmsg("citus_internal_database_command() can only be used " - "for CREATE DATABASE command by " - "Citus."))); + "for CREATE DATABASE command by Citus."))); } /* rollback GUCs to the state before this session */ From a956786bd8fb45e64152a1f2601ee2221bf3164b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 18:59:48 +0300 Subject: [PATCH 144/180] Update src/backend/distributed/deparser/deparse_database_stmts.c Co-authored-by: Onur Tirtir --- src/backend/distributed/deparser/deparse_database_stmts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index e726fa84529..5e7fa54169f 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -322,7 +322,7 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) option->defname))); } - /*if it is the last option then append with ")" */ + /* if it is the last option then append with ")" */ if (option == llast(stmt->options)) { appendStringInfo(buf, " )"); From 1e044a217d9fb5ed6e2aa57f773dfdbc0f77b88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 19:00:01 +0300 Subject: [PATCH 145/180] Update src/backend/distributed/deparser/deparse_database_stmts.c Co-authored-by: Onur Tirtir --- src/backend/distributed/deparser/deparse_database_stmts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 5e7fa54169f..5878d7ea8e2 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -301,7 +301,7 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) foreach_ptr(option, stmt->options) { - /*if it is the first option then append with "WITH" else append with "," */ + /* if it is the first option then append with "WITH" else append with "," */ if (option == linitial(stmt->options)) { appendStringInfo(buf, " WITH ( "); From 77801016192efbb57b78077082109680dc4b6e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Mon, 13 Nov 2023 19:00:40 +0300 Subject: [PATCH 146/180] Update src/backend/distributed/commands/database.c Co-authored-by: Onur Tirtir --- src/backend/distributed/commands/database.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 0d00bb6018c..f846cee70a9 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -654,8 +654,8 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) /* - * GenerateCreateDatabaseCommandList gets a list of pg_database tuples and returns - * a list of CREATE DATABASE statements for all the databases. + * GenerateCreateDatabaseCommandList returns a list of CREATE DATABASE statements + * for all the databases. * * Commands in the list are wrapped by citus_internal_database_command() UDF * to avoid from transaction block restrictions that apply to database commands From c1e9335fb74ed3d6096c8deac73d6ccd2fe076fc Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 14 Nov 2023 09:01:00 +0300 Subject: [PATCH 147/180] Adds distributed check in metadata syncing --- src/backend/distributed/commands/database.c | 9 ++++ .../create_drop_database_propagation.out | 42 +++++++++++++++++++ .../sql/create_drop_database_propagation.sql | 21 ++++++++++ 3 files changed, 72 insertions(+) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index f846cee70a9..0454122c5a7 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -673,6 +673,15 @@ GenerateCreateDatabaseCommandList(void) { Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName( + NameStr(databaseForm->datname), false); + + /* skip databases that are not distributed */ + if (!IsAnyObjectDistributed(list_make1(dbAddress))) + { + continue; + } + char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); StringInfo outerDbStmt = makeStringInfo(); diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 5ab27ba34a4..4eff2e19b8f 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -495,6 +495,48 @@ SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} (3 rows) +-- test that we won't propagate non-distributed databases in citus_add_node +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SET citus.enable_create_database_propagation TO off; +CREATE DATABASE non_distributed_db; +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers +HINT: You can manually create a database and its extensions on workers. +SET citus.enable_create_database_propagation TO on; +create database distributed_db; +select 1 from citus_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +--non_distributed_db should not be propagated to worker_2 +SELECT * FROM public.check_database_on_all_nodes('non_distributed_db') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "non_distributed_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +--distributed_db should be propagated to worker_2 +SELECT * FROM public.check_database_on_all_nodes('distributed_db') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": null, "datname": "distributed_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "distributed_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "distributed_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +--clean up resources created by this test +drop database distributed_db; +set citus.enable_create_database_propagation TO off; +drop database non_distributed_db; --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 1b2f96da2ef..1fdd4ea772a 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -274,6 +274,27 @@ reset citus.log_remote_commands; SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type; +-- test that we won't propagate non-distributed databases in citus_add_node + +select 1 from citus_remove_node('localhost', :worker_2_port); +SET citus.enable_create_database_propagation TO off; +CREATE DATABASE non_distributed_db; +SET citus.enable_create_database_propagation TO on; +create database distributed_db; + +select 1 from citus_add_node('localhost', :worker_2_port); + +--non_distributed_db should not be propagated to worker_2 +SELECT * FROM public.check_database_on_all_nodes('non_distributed_db') ORDER BY node_type; +--distributed_db should be propagated to worker_2 +SELECT * FROM public.check_database_on_all_nodes('distributed_db') ORDER BY node_type; + +--clean up resources created by this test +drop database distributed_db; + +set citus.enable_create_database_propagation TO off; +drop database non_distributed_db; + --clean up resources created by this test From cf019b858cd0c2f31c907c1eed9cc505b6bb59be Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 14 Nov 2023 10:21:23 +0300 Subject: [PATCH 148/180] Adds static declarations --- src/backend/distributed/commands/database.c | 10 ++++++++-- .../distributed/deparser/deparse_database_stmts.c | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 0454122c5a7..a0972eda1bf 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -62,10 +62,16 @@ typedef struct DatabaseCollationInfo #endif } DatabaseCollationInfo; +static void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); +static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm); +static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid); static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); +static char * GetLocaleProviderString(char datlocprovider); +static char * GetTablespaceName(Oid tablespaceOid); +static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName,bool missingOk); + static Oid get_database_owner(Oid db_oid); -List * PreprocessGrantOnDatabaseStmt(Node *node, const char *queryString, - ProcessUtilityContext processUtilityContext); + /* controlled via GUC */ bool EnableCreateDatabasePropagation = false; diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index 5878d7ea8e2..cd86eedf1a2 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -27,8 +27,12 @@ static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); +static void AppendAlterDatabaseSetStmt(StringInfo buf, AlterDatabaseSetStmt *stmt); static void AppendAlterDatabaseStmt(StringInfo buf, AlterDatabaseStmt *stmt); static void AppendDefElemConnLimit(StringInfo buf, DefElem *def); +static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt); +static void AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt); +static void AppendGrantOnDatabaseStmt(StringInfo buf, GrantStmt *stmt); const DefElemOptionFormat create_database_option_formats[] = { { "owner", " OWNER %s", OPTION_FORMAT_STRING }, From 9a558bdece3dd285bc63d353148c63d75db67e39 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 16:04:26 +0300 Subject: [PATCH 149/180] Adds datacl propagation --- src/backend/distributed/commands/common.c | 2 - src/backend/distributed/commands/database.c | 47 ++++- .../distributed/commands/dependencies.c | 7 + src/backend/distributed/commands/role.c | 2 + .../distributed/metadata/metadata_sync.c | 126 +++++++++++- src/include/distributed/commands.h | 1 + src/include/distributed/metadata_sync.h | 1 + .../create_drop_database_propagation.out | 181 ++++++++++++++++++ .../sql/create_drop_database_propagation.sql | 116 +++++++++++ 9 files changed, 471 insertions(+), 12 deletions(-) diff --git a/src/backend/distributed/commands/common.c b/src/backend/distributed/commands/common.c index 957e2616128..797981d47d3 100644 --- a/src/backend/distributed/commands/common.c +++ b/src/backend/distributed/commands/common.c @@ -14,10 +14,8 @@ #include "postgres.h" #include "catalog/objectaddress.h" -#include "catalog/pg_database.h" #include "catalog/pg_ts_config.h" #include "catalog/pg_ts_dict.h" -#include "commands/dbcommands.h" #include "nodes/parsenodes.h" #include "tcop/utility.h" diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index a0972eda1bf..233997dc938 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -658,6 +658,50 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) return str.data; } +/* + * GrantOnDatabaseDDLCommands returns a list of sql statements to idempotently apply a + * GRANT on distributed databases. + */ + +List * GenerateGrantDatabaseCommandList(void){ + List *grantCommands = NIL; + + Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); + TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); + + HeapTuple tuple = NULL; + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); + + ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName( + NameStr(databaseForm->datname), false); + + /* skip databases that are not distributed */ + if (!IsAnyObjectDistributed(list_make1(dbAddress))) + { + continue; + } + + List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); + + /* append dbGrants into grantCommands*/ + grantCommands = list_concat(grantCommands, dbGrants); + } + + char *grantCommand = NULL; + + foreach_ptr(grantCommand, grantCommands) + { + elog(DEBUG1, "grantCommand: %s", grantCommand); + } + + heap_endscan(scan); + table_close(pgDatabaseRel, AccessShareLock); + + return grantCommands; +} + /* * GenerateCreateDatabaseCommandList returns a list of CREATE DATABASE statements @@ -666,8 +710,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) * Commands in the list are wrapped by citus_internal_database_command() UDF * to avoid from transaction block restrictions that apply to database commands */ -List * -GenerateCreateDatabaseCommandList(void) +List * GenerateCreateDatabaseCommandList(void) { List *commands = NIL; diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index e309ee86c7d..9957bcdfcb0 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -465,6 +465,13 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) List *ownerDDLCommands = DatabaseOwnerDDLCommands(dependency); databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands); } + //TODO: To reviewer: Having a code block for dependency makes sense + // However dependency tree is based on pg metadata; which does not reflect + // actual database dependencies. I added this block just to point out the issue. + // if(EnableCreateDatabasePropagation){ + // List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); + // databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); + // } return databaseDDLCommands; } diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index f3ac7b4fff0..632920f7003 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -513,6 +513,8 @@ GenerateRoleOptionsList(HeapTuple tuple) List * GenerateCreateOrAlterRoleCommand(Oid roleOid) { + elog(LOG,"GenerateCreateOrAlterRoleCommand execution"); + HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index cb74ebcb5c1..d917b57a288 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -123,6 +123,7 @@ static List * GetObjectsForGrantStmt(ObjectType objectType, Oid objectId); static AccessPriv * GetAccessPrivObjectForGrantStmt(char *permission); static List * GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem); +static List * GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem); static List * GenerateGrantOnFunctionQueriesFromAclItem(Oid schemaOid, AclItem *aclItem); static List * GrantOnSequenceDDLCommands(Oid sequenceOid); @@ -154,6 +155,7 @@ static char * RemoteSchemaIdExpressionByName(char *schemaName); static char * RemoteTypeIdExpression(Oid typeId); static char * RemoteCollationIdExpression(Oid colocationId); static char * RemoteTableIdExpression(Oid relationId); +static void SendDatabaseGrantSyncCommands(MetadataSyncContext *context); PG_FUNCTION_INFO_V1(start_metadata_sync_to_all_nodes); @@ -2046,6 +2048,80 @@ GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem) return queries; } +List * +GrantOnDatabaseDDLCommands(Oid databaseOid) +{ + HeapTuple databaseTuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(databaseOid)); + bool isNull = true; + Datum aclDatum = SysCacheGetAttr(DATABASEOID, databaseTuple, Anum_pg_database_datacl, + &isNull); + if (isNull) + { + ReleaseSysCache(databaseTuple); + return NIL; + } + Acl *acl = DatumGetAclPCopy(aclDatum); + AclItem *aclDat = ACL_DAT(acl); + int aclNum = ACL_NUM(acl); + List *commands = NIL; + + ReleaseSysCache(databaseTuple); + + for (int i = 0; i < aclNum; i++) + { + commands = list_concat(commands, + GenerateGrantOnDatabaseFromAclItem( + databaseOid,&aclDat[i])); + } + + return commands; +} + + +List * +GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem) +{ + AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_DATABASE; + AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_DATABASE; + + /* + * seems unlikely but we check if there is a grant option in the list without the actual permission + */ + Assert(!(grants & ACL_CONNECT) || (permissions & ACL_CONNECT)); + Assert(!(grants & ACL_CREATE) || (permissions & ACL_CREATE)); + Assert(!(grants & ACL_CREATE_TEMP) || (permissions & ACL_CREATE_TEMP)); + Oid granteeOid = aclItem->ai_grantee; + List *queries = NIL; + + queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor)); + + if (permissions & ACL_CONNECT) + { + char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( + OBJECT_DATABASE ,granteeOid, databaseOid, "CONNECT", + grants & ACL_CONNECT)); + queries = lappend(queries, query); + } + if (permissions & ACL_CREATE) + { + char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( + OBJECT_DATABASE, granteeOid, databaseOid, "CREATE", + grants & ACL_CREATE)); + queries = lappend(queries, query); + } + if (permissions & ACL_CREATE_TEMP) + { + char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( + OBJECT_DATABASE, granteeOid, databaseOid, "TEMPORARY", + grants & ACL_CREATE_TEMP)); + queries = lappend(queries, query); + } + + queries = lappend(queries, "RESET ROLE"); + + return queries; +} + /* * GenerateGrantStmtForRights is the function for creating GrantStmt's for all @@ -2120,6 +2196,11 @@ GetObjectsForGrantStmt(ObjectType objectType, Oid objectId) return list_make1(sequence); } + case OBJECT_DATABASE: + { + return list_make1(makeString(get_database_name(objectId))); + } + default: { elog(ERROR, "unsupported object type for GRANT"); @@ -4563,13 +4644,6 @@ PropagateNodeWideObjectsCommandList(void) /* collect all commands */ List *ddlCommands = NIL; - if (EnableCreateDatabasePropagation) - { - /* get commands for database creation */ - List *createDatabaseCommands = GenerateCreateDatabaseCommandList(); - ddlCommands = list_concat(ddlCommands, createDatabaseCommands); - } - if (EnableAlterRoleSetPropagation) { /* @@ -4580,6 +4654,13 @@ PropagateNodeWideObjectsCommandList(void) ddlCommands = list_concat(ddlCommands, alterRoleSetCommands); } + if (EnableCreateDatabasePropagation) + { + /* get commands for database creation */ + List *createDatabaseCommands = GenerateCreateDatabaseCommandList(); + ddlCommands = list_concat(ddlCommands, createDatabaseCommands); + } + return ddlCommands; } @@ -4611,7 +4692,7 @@ SyncDistributedObjects(MetadataSyncContext *context) Assert(ShouldPropagate()); - /* Send systemwide objects, only roles for now */ + /* send systemwide objects; i.e. roles and databases for now */ SendNodeWideObjectsSyncCommands(context); /* @@ -4651,6 +4732,12 @@ SyncDistributedObjects(MetadataSyncContext *context) * those tables. */ SendInterTableRelationshipCommands(context); + + /* + * After creation of databases and roles, send the grant database commands + * to the workers. + */ + SendDatabaseGrantSyncCommands(context); } @@ -4675,6 +4762,29 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) SendOrCollectCommandListToActivatedNodes(context, commandList); } +/* + * SendDatabaseGrantSyncCommands sends database grants to roles to workers with + * transactional or nontransactional mode according to transactionMode inside + * metadataSyncContext. + * This function is called after SendNodeWideObjectsSyncCommands and SendDependencyCreationCommands + * because we need both databases and roles to be created on the worker. + */ +static void +SendDatabaseGrantSyncCommands(MetadataSyncContext *context) +{ + /* propagate node wide objects. It includes only roles for now. */ + List *commandList = GenerateGrantDatabaseCommandList(); + + if (commandList == NIL) + { + return; + } + + commandList = lcons(DISABLE_DDL_PROPAGATION, commandList); + commandList = lappend(commandList, ENABLE_DDL_PROPAGATION); + SendOrCollectCommandListToActivatedNodes(context, commandList); +} + /* * SendShellTableDeletionCommands sends sequence, and shell table deletion diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 4c47d3ecd21..aaee98d1c2e 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -244,6 +244,7 @@ extern List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk, extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * GenerateCreateDatabaseCommandList(void); +extern List * GenerateGrantDatabaseCommandList(void); /* domain.c - forward declarations */ diff --git a/src/include/distributed/metadata_sync.h b/src/include/distributed/metadata_sync.h index 237df363a13..7b993ec31d4 100644 --- a/src/include/distributed/metadata_sync.h +++ b/src/include/distributed/metadata_sync.h @@ -107,6 +107,7 @@ extern char * ColocationIdUpdateCommand(Oid relationId, uint32 colocationId); extern char * CreateSchemaDDLCommand(Oid schemaId); extern List * GrantOnSchemaDDLCommands(Oid schemaId); extern List * GrantOnFunctionDDLCommands(Oid functionOid); +extern List * GrantOnDatabaseDDLCommands(Oid databaseOid); extern List * GrantOnForeignServerDDLCommands(Oid serverId); extern List * GenerateGrantOnForeignServerQueriesFromAclItem(Oid serverId, AclItem *aclItem); diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 4eff2e19b8f..23184ebca8d 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -487,6 +487,7 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing DROP DATABASE db_force_test WITH ( FORCE ) DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx reset citus.log_remote_commands; +reset citus.grep_remote_commands; SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type; node_type | result --------------------------------------------------------------------- @@ -537,6 +538,186 @@ SELECT * FROM public.check_database_on_all_nodes('distributed_db') ORDER BY node drop database distributed_db; set citus.enable_create_database_propagation TO off; drop database non_distributed_db; +-- test role grants on DATABASE in metadata sync +SET citus.enable_create_database_propagation TO on; +CREATE ROLE db_role_grants_test_role_exists_on_node_2; +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +CREATE DATABASE db_role_grants_test; +revoke connect,temp,temporary on database db_role_grants_test from public; +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE ROLE%'; +CREATE ROLE db_role_grants_test_role_missing_on_node_2; +NOTICE: issuing SELECT worker_create_or_alter_role('db_role_grants_test_role_missing_on_node_2', 'CREATE ROLE db_role_grants_test_role_missing_on_node_2', 'ALTER ROLE db_role_grants_test_role_missing_on_node_2') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +RESET citus.log_remote_commands ; +RESET citus.grep_remote_commands; +-- check the privileges before grant +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + f +(2 rows) + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%GRANT%'; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +select 1 from citus_add_node('localhost', :worker_2_port); +NOTICE: issuing CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION pg_database_owner;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO pg_database_owner;;GRANT CREATE ON SCHEMA public TO pg_database_owner;;RESET ROLE;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO PUBLIC;;RESET ROLE +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing CREATE SCHEMA IF NOT EXISTS information_schema AUTHORIZATION postgres;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO postgres;;GRANT CREATE ON SCHEMA information_schema TO postgres;;RESET ROLE;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO PUBLIC;;RESET ROLE +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO postgres;;GRANT CREATE ON DATABASE db_role_grants_test TO postgres;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO postgres;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;RESET ROLE;SET citus.enable_ddl_propagation TO 'on' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t + t +(3 rows) + +DROP DATABASE db_role_grants_test; +DROP ROLE db_role_grants_test_role_exists_on_node_2; +DROP ROLE db_role_grants_test_role_missing_on_node_2; --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 1fdd4ea772a..bc53f87df84 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -271,6 +271,7 @@ set citus.grep_remote_commands = '%DROP DATABASE%'; drop database db_force_test with (force); reset citus.log_remote_commands; +reset citus.grep_remote_commands; SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type; @@ -296,6 +297,121 @@ set citus.enable_create_database_propagation TO off; drop database non_distributed_db; + +-- test role grants on DATABASE in metadata sync + + + +SET citus.enable_create_database_propagation TO on; + + + +CREATE ROLE db_role_grants_test_role_exists_on_node_2; + + +select 1 from citus_remove_node('localhost', :worker_2_port); + +CREATE DATABASE db_role_grants_test; + +revoke connect,temp,temporary on database db_role_grants_test from public; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%CREATE ROLE%'; +CREATE ROLE db_role_grants_test_role_missing_on_node_2; + +RESET citus.log_remote_commands ; +RESET citus.grep_remote_commands; + +-- check the privileges before grant + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%GRANT%'; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; + +select 1 from citus_add_node('localhost', :worker_2_port); + + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') + $$ +) ORDER BY result; + + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT') + $$ +) ORDER BY result; + + +DROP DATABASE db_role_grants_test; +DROP ROLE db_role_grants_test_role_exists_on_node_2; +DROP ROLE db_role_grants_test_role_missing_on_node_2; + + --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. From eb46d9399bce062e6cd01bc6a1cad9c8cd42bde3 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 16:26:14 +0300 Subject: [PATCH 150/180] Fixes warning on compile --- src/backend/distributed/commands/database.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 233997dc938..3570966a204 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -66,7 +66,9 @@ static void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm); static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid); static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); +#if PG_VERSION_NUM >= PG_VERSION_15 static char * GetLocaleProviderString(char datlocprovider); +#endif static char * GetTablespaceName(Oid tablespaceOid); static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName,bool missingOk); From 6f838e6e8849d11d6e59962678050771a3ce443a Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 16:34:02 +0300 Subject: [PATCH 151/180] Fixes indentation --- src/backend/distributed/commands/database.c | 17 +++++++++++------ src/backend/distributed/commands/dependencies.c | 14 +++++++------- src/backend/distributed/commands/role.c | 2 +- .../distributed/metadata/metadata_sync.c | 15 ++++++++++----- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 3570966a204..71e33b096df 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -63,14 +63,16 @@ typedef struct DatabaseCollationInfo } DatabaseCollationInfo; static void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); -static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm); +static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database + databaseForm); static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid); static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); #if PG_VERSION_NUM >= PG_VERSION_15 static char * GetLocaleProviderString(char datlocprovider); #endif static char * GetTablespaceName(Oid tablespaceOid); -static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName,bool missingOk); +static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, bool + missingOk); static Oid get_database_owner(Oid db_oid); @@ -660,12 +662,14 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) return str.data; } + /* * GrantOnDatabaseDDLCommands returns a list of sql statements to idempotently apply a * GRANT on distributed databases. */ - -List * GenerateGrantDatabaseCommandList(void){ +List * +GenerateGrantDatabaseCommandList(void) +{ List *grantCommands = NIL; Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); @@ -685,7 +689,7 @@ List * GenerateGrantDatabaseCommandList(void){ continue; } - List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); + List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); /* append dbGrants into grantCommands*/ grantCommands = list_concat(grantCommands, dbGrants); @@ -712,7 +716,8 @@ List * GenerateGrantDatabaseCommandList(void){ * Commands in the list are wrapped by citus_internal_database_command() UDF * to avoid from transaction block restrictions that apply to database commands */ -List * GenerateCreateDatabaseCommandList(void) +List * +GenerateCreateDatabaseCommandList(void) { List *commands = NIL; diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 9957bcdfcb0..3c279f1e6fd 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -465,13 +465,13 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) List *ownerDDLCommands = DatabaseOwnerDDLCommands(dependency); databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands); } - //TODO: To reviewer: Having a code block for dependency makes sense - // However dependency tree is based on pg metadata; which does not reflect - // actual database dependencies. I added this block just to point out the issue. - // if(EnableCreateDatabasePropagation){ - // List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); - // databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); - // } + /*TODO: To reviewer: Having a code block for dependency makes sense */ + /* However dependency tree is based on pg metadata; which does not reflect */ + /* actual database dependencies. I added this block just to point out the issue. */ + /* if(EnableCreateDatabasePropagation){ */ + /* List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); */ + /* databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); */ + /* } */ return databaseDDLCommands; } diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 632920f7003..0293d6a7b63 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -513,7 +513,7 @@ GenerateRoleOptionsList(HeapTuple tuple) List * GenerateCreateOrAlterRoleCommand(Oid roleOid) { - elog(LOG,"GenerateCreateOrAlterRoleCommand execution"); + elog(LOG, "GenerateCreateOrAlterRoleCommand execution"); HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index d917b57a288..089094fac8a 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -2048,6 +2048,7 @@ GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem) return queries; } + List * GrantOnDatabaseDDLCommands(Oid databaseOid) { @@ -2071,7 +2072,7 @@ GrantOnDatabaseDDLCommands(Oid databaseOid) { commands = list_concat(commands, GenerateGrantOnDatabaseFromAclItem( - databaseOid,&aclDat[i])); + databaseOid, &aclDat[i])); } return commands; @@ -2098,21 +2099,24 @@ GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem) if (permissions & ACL_CONNECT) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( - OBJECT_DATABASE ,granteeOid, databaseOid, "CONNECT", + OBJECT_DATABASE, granteeOid, databaseOid, + "CONNECT", grants & ACL_CONNECT)); queries = lappend(queries, query); } if (permissions & ACL_CREATE) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( - OBJECT_DATABASE, granteeOid, databaseOid, "CREATE", + OBJECT_DATABASE, granteeOid, databaseOid, + "CREATE", grants & ACL_CREATE)); queries = lappend(queries, query); } if (permissions & ACL_CREATE_TEMP) { char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights( - OBJECT_DATABASE, granteeOid, databaseOid, "TEMPORARY", + OBJECT_DATABASE, granteeOid, databaseOid, + "TEMPORARY", grants & ACL_CREATE_TEMP)); queries = lappend(queries, query); } @@ -4736,7 +4740,7 @@ SyncDistributedObjects(MetadataSyncContext *context) /* * After creation of databases and roles, send the grant database commands * to the workers. - */ + */ SendDatabaseGrantSyncCommands(context); } @@ -4762,6 +4766,7 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) SendOrCollectCommandListToActivatedNodes(context, commandList); } + /* * SendDatabaseGrantSyncCommands sends database grants to roles to workers with * transactional or nontransactional mode according to transactionMode inside From 931514b0f7631a3b86538efe11ad7b513cee08be Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 16:41:38 +0300 Subject: [PATCH 152/180] Fixes indentation --- src/backend/distributed/commands/dependencies.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 3c279f1e6fd..732d1040974 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -465,6 +465,7 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) List *ownerDDLCommands = DatabaseOwnerDDLCommands(dependency); databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands); } + /*TODO: To reviewer: Having a code block for dependency makes sense */ /* However dependency tree is based on pg metadata; which does not reflect */ /* actual database dependencies. I added this block just to point out the issue. */ From 37681eaed074a2bc26280aa644b600e05eed1f36 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 17:01:42 +0300 Subject: [PATCH 153/180] Fixes comment on code --- src/backend/distributed/commands/dependencies.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 732d1040974..c15f9fba4cb 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -468,11 +468,17 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) /*TODO: To reviewer: Having a code block for dependency makes sense */ /* However dependency tree is based on pg metadata; which does not reflect */ - /* actual database dependencies. I added this block just to point out the issue. */ - /* if(EnableCreateDatabasePropagation){ */ - /* List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); */ - /* databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); */ - /* } */ + + /* actual database dependencies. I added this block just to point out the issue. + */ + + /* + * if(EnableCreateDatabasePropagation){ + * List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); + * databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); + * + * } + */ return databaseDDLCommands; } From 79392673267b3a11ff34d19b39d8e56b3c9ed312 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 17:28:50 +0300 Subject: [PATCH 154/180] Fixes flakiness of test --- .../regress/expected/create_drop_database_propagation.out | 8 ++------ src/test/regress/sql/create_drop_database_propagation.sql | 5 +++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 23184ebca8d..993abee1da6 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -631,13 +631,9 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +RESET citus.log_remote_commands; +RESET citus.grep_remote_commands; select 1 from citus_add_node('localhost', :worker_2_port); -NOTICE: issuing CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION pg_database_owner;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO pg_database_owner;;GRANT CREATE ON SCHEMA public TO pg_database_owner;;RESET ROLE;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO PUBLIC;;RESET ROLE -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing CREATE SCHEMA IF NOT EXISTS information_schema AUTHORIZATION postgres;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO postgres;;GRANT CREATE ON SCHEMA information_schema TO postgres;;RESET ROLE;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO PUBLIC;;RESET ROLE -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SET citus.enable_ddl_propagation TO 'off';SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO postgres;;GRANT CREATE ON DATABASE db_role_grants_test TO postgres;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO postgres;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;RESET ROLE;SET citus.enable_ddl_propagation TO 'on' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx ?column? --------------------------------------------------------------------- 1 diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index bc53f87df84..c151f463882 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -366,9 +366,14 @@ set citus.grep_remote_commands = '%GRANT%'; grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; + +RESET citus.log_remote_commands; +RESET citus.grep_remote_commands; + select 1 from citus_add_node('localhost', :worker_2_port); + SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') From deee8e53dd3cd4ad864007f64d56fd6c41c92b61 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 17:33:14 +0300 Subject: [PATCH 155/180] Adds grant to public --- .../regress/expected/create_drop_database_propagation.out | 3 ++- src/test/regress/sql/create_drop_database_propagation.sql | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 993abee1da6..ac3b9a1c208 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -548,7 +548,7 @@ select 1 from citus_remove_node('localhost', :worker_2_port); (1 row) CREATE DATABASE db_role_grants_test; -revoke connect,temp,temporary on database db_role_grants_test from public; +revoke connect,temp,temporary,create on database db_role_grants_test from public; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE ROLE%'; CREATE ROLE db_role_grants_test_role_missing_on_node_2; @@ -711,6 +711,7 @@ SELECT result from run_command_on_all_nodes( t (3 rows) +grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; DROP ROLE db_role_grants_test_role_exists_on_node_2; DROP ROLE db_role_grants_test_role_missing_on_node_2; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index c151f463882..5eed4ad2449 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -313,7 +313,7 @@ select 1 from citus_remove_node('localhost', :worker_2_port); CREATE DATABASE db_role_grants_test; -revoke connect,temp,temporary on database db_role_grants_test from public; +revoke connect,temp,temporary,create on database db_role_grants_test from public; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE ROLE%'; @@ -411,12 +411,15 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; DROP ROLE db_role_grants_test_role_exists_on_node_2; DROP ROLE db_role_grants_test_role_missing_on_node_2; + + --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. From dc48833679004076524baa5949413464907d2060 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 18:48:46 +0300 Subject: [PATCH 156/180] Adds tests for non-distributed database --- .../create_drop_database_propagation.out | 229 ++++++++++++++++-- .../sql/create_drop_database_propagation.sql | 126 +++++++++- 2 files changed, 329 insertions(+), 26 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index ac3b9a1c208..fa38a1a6958 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -539,6 +539,30 @@ drop database distributed_db; set citus.enable_create_database_propagation TO off; drop database non_distributed_db; -- test role grants on DATABASE in metadata sync +SELECT result from run_command_on_all_nodes( + $$ + create database db_role_grants_test_non_distributed + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + CREATE DATABASE + CREATE DATABASE + CREATE DATABASE +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + ERROR: operation is not allowed on this node + ERROR: operation is not allowed on this node + REVOKE +(3 rows) + SET citus.enable_create_database_propagation TO on; CREATE ROLE db_role_grants_test_role_exists_on_node_2; select 1 from citus_remove_node('localhost', :worker_2_port); @@ -556,7 +580,22 @@ NOTICE: issuing SELECT worker_create_or_alter_role('db_role_grants_test_role_mi DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx RESET citus.log_remote_commands ; RESET citus.grep_remote_commands; --- check the privileges before grant +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%GRANT%'; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_exists_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test_non_distributed TO db_role_grants_test_role_exists_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; +NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test_non_distributed TO db_role_grants_test_role_missing_on_node_2; +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- check the privileges before add_node for database db_role_grants_test, +-- role db_role_grants_test_role_exists_on_node_2 SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') @@ -564,8 +603,8 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t (2 rows) SELECT result from run_command_on_all_nodes( @@ -575,8 +614,8 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t (2 rows) SELECT result from run_command_on_all_nodes( @@ -586,10 +625,12 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t (2 rows) +-- check the privileges before add_node for database db_role_grants_test, +-- role db_role_grants_test_role_missing_on_node_2 SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') @@ -597,8 +638,8 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t (2 rows) SELECT result from run_command_on_all_nodes( @@ -608,8 +649,8 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t (2 rows) SELECT result from run_command_on_all_nodes( @@ -619,18 +660,80 @@ SELECT result from run_command_on_all_nodes( ) ORDER BY result; result --------------------------------------------------------------------- - f - f + t + t +(2 rows) + +-- check the privileges before add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_exists_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t +(2 rows) + +-- check the privileges before add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_missing_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + t + t (2 rows) -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%GRANT%'; -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; -NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; -NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2; -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx RESET citus.log_remote_commands; RESET citus.grep_remote_commands; select 1 from citus_add_node('localhost', :worker_2_port); @@ -639,6 +742,8 @@ select 1 from citus_add_node('localhost', :worker_2_port); 1 (1 row) +-- check the privileges after add_node for database db_role_grants_test, +-- role db_role_grants_test_role_exists_on_node_2 SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE') @@ -675,6 +780,8 @@ SELECT result from run_command_on_all_nodes( t (3 rows) +-- check the privileges after add_node for database db_role_grants_test, +-- role db_role_grants_test_role_missing_on_node_2 SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') @@ -711,10 +818,90 @@ SELECT result from run_command_on_all_nodes( t (3 rows) +-- check the privileges after add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_exists_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + +-- check the privileges after add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_missing_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + f + t + t +(3 rows) + grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; DROP ROLE db_role_grants_test_role_exists_on_node_2; +ERROR: role "db_role_grants_test_role_exists_on_node_2" cannot be dropped because some objects depend on it +DETAIL: privileges for database db_role_grants_test_non_distributed DROP ROLE db_role_grants_test_role_missing_on_node_2; +ERROR: role "db_role_grants_test_role_missing_on_node_2" cannot be dropped because some objects depend on it +DETAIL: privileges for database db_role_grants_test_non_distributed --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 5eed4ad2449..727ae02982a 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -300,7 +300,17 @@ drop database non_distributed_db; -- test role grants on DATABASE in metadata sync +SELECT result from run_command_on_all_nodes( + $$ + create database db_role_grants_test_non_distributed + $$ +) ORDER BY result; +SELECT result from run_command_on_all_nodes( + $$ + revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public + $$ +) ORDER BY result; SET citus.enable_create_database_propagation TO on; @@ -322,7 +332,20 @@ CREATE ROLE db_role_grants_test_role_missing_on_node_2; RESET citus.log_remote_commands ; RESET citus.grep_remote_commands; --- check the privileges before grant + + +SET citus.log_remote_commands = true; +set citus.grep_remote_commands = '%GRANT%'; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; + + + +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_exists_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; + +-- check the privileges before add_node for database db_role_grants_test, +-- role db_role_grants_test_role_exists_on_node_2 SELECT result from run_command_on_all_nodes( $$ @@ -342,6 +365,9 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +-- check the privileges before add_node for database db_role_grants_test, +-- role db_role_grants_test_role_missing_on_node_2 + SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') @@ -361,10 +387,47 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -SET citus.log_remote_commands = true; -set citus.grep_remote_commands = '%GRANT%'; -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; +-- check the privileges before add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_exists_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + +-- check the privileges before add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_missing_on_node_2 + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; RESET citus.log_remote_commands; @@ -373,6 +436,8 @@ RESET citus.grep_remote_commands; select 1 from citus_add_node('localhost', :worker_2_port); +-- check the privileges after add_node for database db_role_grants_test, +-- role db_role_grants_test_role_exists_on_node_2 SELECT result from run_command_on_all_nodes( $$ @@ -392,6 +457,9 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +-- check the privileges after add_node for database db_role_grants_test, +-- role db_role_grants_test_role_missing_on_node_2 + SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE') @@ -411,9 +479,57 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; +-- check the privileges after add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_exists_on_node_2 +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + +-- check the privileges after add_node for database db_role_grants_test_non_distributed, +-- role db_role_grants_test_role_missing_on_node_2 + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CREATE') + $$ +) ORDER BY result; + + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') + $$ +) ORDER BY result; + +SELECT result from run_command_on_all_nodes( + $$ + select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'CONNECT') + $$ +) ORDER BY result; + grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; + +SELECT result from run_command_on_all_nodes( + $$ + drop database db_role_grants_test_non_distributed + $$ +) ORDER BY result; DROP ROLE db_role_grants_test_role_exists_on_node_2; DROP ROLE db_role_grants_test_role_missing_on_node_2; From 0f1273f6ad737aab5b1d117cfbcd45f2e2442a58 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 19:03:16 +0300 Subject: [PATCH 157/180] Updates test output --- .../create_drop_database_propagation.out | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index fa38a1a6958..84ad9f7b5db 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -896,12 +896,20 @@ SELECT result from run_command_on_all_nodes( grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; +SELECT result from run_command_on_all_nodes( + $$ + drop database db_role_grants_test_non_distributed + $$ +) ORDER BY result; + result +--------------------------------------------------------------------- + DROP DATABASE + DROP DATABASE + DROP DATABASE +(3 rows) + DROP ROLE db_role_grants_test_role_exists_on_node_2; -ERROR: role "db_role_grants_test_role_exists_on_node_2" cannot be dropped because some objects depend on it -DETAIL: privileges for database db_role_grants_test_non_distributed DROP ROLE db_role_grants_test_role_missing_on_node_2; -ERROR: role "db_role_grants_test_role_missing_on_node_2" cannot be dropped because some objects depend on it -DETAIL: privileges for database db_role_grants_test_non_distributed --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( @@ -917,3 +925,4 @@ SELECT result FROM run_command_on_all_nodes( (3 rows) drop user create_drop_db_test_user; + From 11e7c94e2f6f62c08f6476e4d45b9d5a64ce10a1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 19:32:00 +0300 Subject: [PATCH 158/180] Removes extra line in test --- src/test/regress/expected/create_drop_database_propagation.out | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index 84ad9f7b5db..b0831aa5540 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -925,4 +925,3 @@ SELECT result FROM run_command_on_all_nodes( (3 rows) drop user create_drop_db_test_user; - From 144ede3bf0fa6b730590742da8497ff5860632ee Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 20:45:13 +0300 Subject: [PATCH 159/180] Adds GUC check for db grant sync --- src/backend/distributed/metadata/metadata_sync.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 089094fac8a..af352088823 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4741,7 +4741,9 @@ SyncDistributedObjects(MetadataSyncContext *context) * After creation of databases and roles, send the grant database commands * to the workers. */ - SendDatabaseGrantSyncCommands(context); + if (EnableCreateDatabasePropagation){ + SendDatabaseGrantSyncCommands(context); + } } From e18c4400dcccc48200d823df461a6733fb7e9d10 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 20:48:17 +0300 Subject: [PATCH 160/180] Removes unnecessary logs --- src/backend/distributed/commands/database.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 71e33b096df..b988122ef3c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -695,13 +695,6 @@ GenerateGrantDatabaseCommandList(void) grantCommands = list_concat(grantCommands, dbGrants); } - char *grantCommand = NULL; - - foreach_ptr(grantCommand, grantCommands) - { - elog(DEBUG1, "grantCommand: %s", grantCommand); - } - heap_endscan(scan); table_close(pgDatabaseRel, AccessShareLock); From cd40380d80807d0f0ddec8cfcd3be6665e13b639 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 20:51:40 +0300 Subject: [PATCH 161/180] resets enable_create_database_propagation --- src/test/regress/sql/create_drop_database_propagation.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 727ae02982a..cdd7cb3e0cf 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -546,3 +546,4 @@ SELECT result FROM run_command_on_all_nodes( ); drop user create_drop_db_test_user; +reset enable_create_database_propagation; From 9b6511ad10d9fea81c0a707fc50715582e05fddc Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 20:57:27 +0300 Subject: [PATCH 162/180] Fixes indentation --- src/backend/distributed/metadata/metadata_sync.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index af352088823..d31ccde3d8a 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4741,7 +4741,8 @@ SyncDistributedObjects(MetadataSyncContext *context) * After creation of databases and roles, send the grant database commands * to the workers. */ - if (EnableCreateDatabasePropagation){ + if (EnableCreateDatabasePropagation) + { SendDatabaseGrantSyncCommands(context); } } From feb609868e2648ac638b86e5742f00adfded91f9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 21:12:45 +0300 Subject: [PATCH 163/180] Fixes flag name --- src/test/regress/expected/create_drop_database_propagation.out | 1 + src/test/regress/sql/create_drop_database_propagation.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index b0831aa5540..bc3d5314f05 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -925,3 +925,4 @@ SELECT result FROM run_command_on_all_nodes( (3 rows) drop user create_drop_db_test_user; +reset citus.enable_create_database_propagation; diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index cdd7cb3e0cf..82e179591db 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -546,4 +546,4 @@ SELECT result FROM run_command_on_all_nodes( ); drop user create_drop_db_test_user; -reset enable_create_database_propagation; +reset citus.enable_create_database_propagation; From 772a13dc7c13983cb9b7f11d0a6518ea46fc26a4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 21:32:01 +0300 Subject: [PATCH 164/180] Removes SendDatabaseGrantSyncCommands --- src/backend/distributed/metadata/metadata_sync.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index d31ccde3d8a..2a1f826f424 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4741,10 +4741,9 @@ SyncDistributedObjects(MetadataSyncContext *context) * After creation of databases and roles, send the grant database commands * to the workers. */ - if (EnableCreateDatabasePropagation) - { - SendDatabaseGrantSyncCommands(context); - } + + SendDatabaseGrantSyncCommands(context); + } From a68587391c766788c95984d543f349e16bd6e6fc Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 21:32:28 +0300 Subject: [PATCH 165/180] Fixes indentation --- src/backend/distributed/metadata/metadata_sync.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 2a1f826f424..82267ac0290 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4743,7 +4743,6 @@ SyncDistributedObjects(MetadataSyncContext *context) */ SendDatabaseGrantSyncCommands(context); - } From bb2b7ae9da0af87297f49b8b5b93ea970116e2a6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 23:02:21 +0300 Subject: [PATCH 166/180] Adds logs for test --- src/backend/distributed/commands/database.c | 1 + src/test/regress/sql/multi_metadata_sync.sql | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index b988122ef3c..4a1cb642872 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -688,6 +688,7 @@ GenerateGrantDatabaseCommandList(void) { continue; } + elog(NOTICE, "Granting on database %s", NameStr(databaseForm->datname)); List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql index 1b8043cdd6d..4c72de7adfb 100644 --- a/src/test/regress/sql/multi_metadata_sync.sql +++ b/src/test/regress/sql/multi_metadata_sync.sql @@ -47,8 +47,15 @@ ALTER ROLE CURRENT_USER WITH PASSWORD 'dummypassword'; -- Show that, with no MX tables, activate node snapshot contains only the delete commands, -- pg_dist_node entries, pg_dist_object entries and roles. + +select pdo.*, pd.datname +from pg_dist_object pdo + left outer join pg_database pd on pdo.objid = pd.oid; + SELECT unnest(activate_node_snapshot()) order by 1; + + -- Create a test table with constraints and SERIAL and default from user defined sequence CREATE SEQUENCE user_defined_seq; CREATE TABLE mx_test_table (col_1 int UNIQUE, col_2 text NOT NULL, col_3 BIGSERIAL, col_4 BIGINT DEFAULT nextval('user_defined_seq')); From b474d98e5c670d09f10596a121661952b6fcfd3f Mon Sep 17 00:00:00 2001 From: gindibay Date: Wed, 15 Nov 2023 23:09:35 +0300 Subject: [PATCH 167/180] Removes log --- src/backend/distributed/commands/database.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 4a1cb642872..b988122ef3c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -688,7 +688,6 @@ GenerateGrantDatabaseCommandList(void) { continue; } - elog(NOTICE, "Granting on database %s", NameStr(databaseForm->datname)); List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); From 95572c5adaf437156e1580a82477fe3c1633a189 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 02:58:56 +0300 Subject: [PATCH 168/180] Adds EnableCreateDatabasePropagation flag check --- src/backend/distributed/commands/role.c | 1 - .../distributed/metadata/metadata_sync.c | 23 +++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 0293d6a7b63..3c56e3fd725 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -513,7 +513,6 @@ GenerateRoleOptionsList(HeapTuple tuple) List * GenerateCreateOrAlterRoleCommand(Oid roleOid) { - elog(LOG, "GenerateCreateOrAlterRoleCommand execution"); HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 82267ac0290..77e0edd1176 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4771,24 +4771,27 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) /* * SendDatabaseGrantSyncCommands sends database grants to roles to workers with * transactional or nontransactional mode according to transactionMode inside - * metadataSyncContext. + * metadataSyncContext in case of EnableCreateDatabasePropagation GUC set. * This function is called after SendNodeWideObjectsSyncCommands and SendDependencyCreationCommands * because we need both databases and roles to be created on the worker. + * */ static void SendDatabaseGrantSyncCommands(MetadataSyncContext *context) { - /* propagate node wide objects. It includes only roles for now. */ - List *commandList = GenerateGrantDatabaseCommandList(); + if(EnableCreateDatabasePropagation){ + /* propagate node wide objects. It includes only roles for now. */ + List *commandList = GenerateGrantDatabaseCommandList(); - if (commandList == NIL) - { - return; - } + if (commandList == NIL) + { + return; + } - commandList = lcons(DISABLE_DDL_PROPAGATION, commandList); - commandList = lappend(commandList, ENABLE_DDL_PROPAGATION); - SendOrCollectCommandListToActivatedNodes(context, commandList); + commandList = lcons(DISABLE_DDL_PROPAGATION, commandList); + commandList = lappend(commandList, ENABLE_DDL_PROPAGATION); + SendOrCollectCommandListToActivatedNodes(context, commandList); + } } From 8443ae36e70db11bf978e2baec960bea8ff23070 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 02:59:32 +0300 Subject: [PATCH 169/180] Fixes indentation --- src/backend/distributed/commands/role.c | 1 - src/backend/distributed/metadata/metadata_sync.c | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 3c56e3fd725..f3ac7b4fff0 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -513,7 +513,6 @@ GenerateRoleOptionsList(HeapTuple tuple) List * GenerateCreateOrAlterRoleCommand(Oid roleOid) { - HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid)); Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple)); diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 77e0edd1176..ce2201903fb 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4779,7 +4779,8 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) static void SendDatabaseGrantSyncCommands(MetadataSyncContext *context) { - if(EnableCreateDatabasePropagation){ + if (EnableCreateDatabasePropagation) + { /* propagate node wide objects. It includes only roles for now. */ List *commandList = GenerateGrantDatabaseCommandList(); From 56bc813bd0471f75e6111d40cb7d36d57c8cf891 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 05:13:10 +0300 Subject: [PATCH 170/180] Adds check_database_on_all_nodes --- .../regress/expected/multi_test_helpers.out | 55 +++++++++++++++++++ src/test/regress/sql/multi_metadata_sync.sql | 7 --- src/test/regress/sql/multi_test_helpers.sql | 55 +++++++++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 4b621b968d5..9822a093106 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -556,3 +556,58 @@ BEGIN ORDER BY node_type; END; $func$ LANGUAGE plpgsql; +$func$ LANGUAGE plpgsql; +-- For all nodes, returns database properties of given database, except +-- oid, datfrozenxid and datminmxid. +-- +-- Also returns whether the node has a pg_dist_object record for the database +-- and whether there are any stale pg_dist_object records for a database. +CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) +RETURNS TABLE (node_type text, result text) +AS $func$ +BEGIN + RETURN QUERY + SELECT + CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' + WHEN (groupid = 0) THEN 'coordinator (remote)' + WHEN (groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'worker node (local)' + ELSE 'worker node (remote)' + END AS node_type, + q2.result + FROM run_command_on_all_nodes( + format( + $$ + SELECT to_jsonb(q.*) + FROM ( + SELECT + ( + SELECT to_jsonb(database_properties.*) + FROM ( + SELECT datname, pa.rolname as database_owner, + pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, + datistemplate, datallowconn, datconnlimit, + pt.spcname AS tablespace, datcollate, datctype, daticulocale, + datcollversion, datacl + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + JOIN pg_tablespace pt ON pd.dattablespace = pt.oid + WHERE datname = '%s' + ) database_properties + ) AS database_properties, + ( + SELECT COUNT(*)=1 + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + ) AS pg_dist_object_record_for_db_exists, + ( + SELECT COUNT(*) > 0 + FROM pg_dist_object + WHERE classid = 1262 AND objid NOT IN (SELECT oid FROM pg_database) + ) AS stale_pg_dist_object_record_for_a_db_exists + ) q + $$, + p_database_name, p_database_name + ) + ) q2 + JOIN pg_dist_node USING (nodeid); +END; +$func$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql index 4c72de7adfb..1b8043cdd6d 100644 --- a/src/test/regress/sql/multi_metadata_sync.sql +++ b/src/test/regress/sql/multi_metadata_sync.sql @@ -47,15 +47,8 @@ ALTER ROLE CURRENT_USER WITH PASSWORD 'dummypassword'; -- Show that, with no MX tables, activate node snapshot contains only the delete commands, -- pg_dist_node entries, pg_dist_object entries and roles. - -select pdo.*, pd.datname -from pg_dist_object pdo - left outer join pg_database pd on pdo.objid = pd.oid; - SELECT unnest(activate_node_snapshot()) order by 1; - - -- Create a test table with constraints and SERIAL and default from user defined sequence CREATE SEQUENCE user_defined_seq; CREATE TABLE mx_test_table (col_1 int UNIQUE, col_2 text NOT NULL, col_3 BIGSERIAL, col_4 BIGINT DEFAULT nextval('user_defined_seq')); diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 7f0346d1421..3aa86b86c05 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -581,3 +581,58 @@ BEGIN ORDER BY node_type; END; $func$ LANGUAGE plpgsql; + +-- For all nodes, returns database properties of given database, except +-- oid, datfrozenxid and datminmxid. +-- +-- Also returns whether the node has a pg_dist_object record for the database +-- and whether there are any stale pg_dist_object records for a database. +CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) +RETURNS TABLE (node_type text, result text) +AS $func$ +BEGIN + RETURN QUERY + SELECT + CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' + WHEN (groupid = 0) THEN 'coordinator (remote)' + WHEN (groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'worker node (local)' + ELSE 'worker node (remote)' + END AS node_type, + q2.result + FROM run_command_on_all_nodes( + format( + $$ + SELECT to_jsonb(q.*) + FROM ( + SELECT + ( + SELECT to_jsonb(database_properties.*) + FROM ( + SELECT datname, pa.rolname as database_owner, + pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, + datistemplate, datallowconn, datconnlimit, + pt.spcname AS tablespace, datcollate, datctype, daticulocale, + datcollversion, datacl + FROM pg_database pd + JOIN pg_authid pa ON pd.datdba = pa.oid + JOIN pg_tablespace pt ON pd.dattablespace = pt.oid + WHERE datname = '%s' + ) database_properties + ) AS database_properties, + ( + SELECT COUNT(*)=1 + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + ) AS pg_dist_object_record_for_db_exists, + ( + SELECT COUNT(*) > 0 + FROM pg_dist_object + WHERE classid = 1262 AND objid NOT IN (SELECT oid FROM pg_database) + ) AS stale_pg_dist_object_record_for_a_db_exists + ) q + $$, + p_database_name, p_database_name + ) + ) q2 + JOIN pg_dist_node USING (nodeid); +END; +$func$ LANGUAGE plpgsql; From a6e16252d1631b967ff5f216657197ac488dfc6e Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 05:34:41 +0300 Subject: [PATCH 171/180] Removes extra line --- src/test/regress/expected/multi_test_helpers.out | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 9822a093106..4f21258cf38 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -556,7 +556,6 @@ BEGIN ORDER BY node_type; END; $func$ LANGUAGE plpgsql; -$func$ LANGUAGE plpgsql; -- For all nodes, returns database properties of given database, except -- oid, datfrozenxid and datminmxid. -- From ef2a47e882666622a42cc81cc2622fa4f7fef85e Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 07:20:59 +0300 Subject: [PATCH 172/180] Fixes function after merge --- src/test/regress/sql/multi_test_helpers.sql | 30 ++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 3aa86b86c05..e67b782a524 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -590,7 +590,22 @@ $func$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) RETURNS TABLE (node_type text, result text) AS $func$ +DECLARE + pg_ge_15_options text := ''; + pg_ge_16_options text := ''; BEGIN + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'datlocprovider') THEN + pg_ge_15_options := ', daticulocale, datcollversion, datlocprovider'; + ELSE + pg_ge_15_options := $$, null as daticulocale, null as datcollversion, 'c' as datlocprovider$$; + END IF; + + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'daticurules') THEN + pg_ge_16_options := ', daticurules'; + ELSE + pg_ge_16_options := ', null as daticurules'; + END IF; + RETURN QUERY SELECT CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' @@ -609,19 +624,20 @@ BEGIN SELECT to_jsonb(database_properties.*) FROM ( SELECT datname, pa.rolname as database_owner, - pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, - datistemplate, datallowconn, datconnlimit, - pt.spcname AS tablespace, datcollate, datctype, daticulocale, - datcollversion, datacl + pg_encoding_to_char(pd.encoding) as encoding, + datistemplate, datallowconn, datconnlimit, datacl, + pt.spcname AS tablespace, datcollate, datctype + %2$s -- >= pg15 options + %3$s -- >= pg16 options FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid JOIN pg_tablespace pt ON pd.dattablespace = pt.oid - WHERE datname = '%s' + WHERE datname = '%1$s' ) database_properties ) AS database_properties, ( SELECT COUNT(*)=1 - FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%1$s') ) AS pg_dist_object_record_for_db_exists, ( SELECT COUNT(*) > 0 @@ -630,7 +646,7 @@ BEGIN ) AS stale_pg_dist_object_record_for_a_db_exists ) q $$, - p_database_name, p_database_name + p_database_name, pg_ge_15_options, pg_ge_16_options ) ) q2 JOIN pg_dist_node USING (nodeid); From a948358d630dc715005f432d03be0d0ada8c8982 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 08:01:44 +0300 Subject: [PATCH 173/180] Fixes multi_test_helpers --- .../regress/expected/multi_test_helpers.out | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 4f21258cf38..ea02a1cf830 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -555,7 +555,6 @@ BEGIN JOIN pg_dist_node USING (nodeid) ORDER BY node_type; END; -$func$ LANGUAGE plpgsql; -- For all nodes, returns database properties of given database, except -- oid, datfrozenxid and datminmxid. -- @@ -564,7 +563,22 @@ $func$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION check_database_on_all_nodes(p_database_name text) RETURNS TABLE (node_type text, result text) AS $func$ +DECLARE + pg_ge_15_options text := ''; + pg_ge_16_options text := ''; BEGIN + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'datlocprovider') THEN + pg_ge_15_options := ', daticulocale, datcollversion, datlocprovider'; + ELSE + pg_ge_15_options := $$, null as daticulocale, null as datcollversion, 'c' as datlocprovider$$; + END IF; + + IF EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'pg_database'::regclass AND attname = 'daticurules') THEN + pg_ge_16_options := ', daticurules'; + ELSE + pg_ge_16_options := ', null as daticurules'; + END IF; + RETURN QUERY SELECT CASE WHEN (groupid = 0 AND groupid = (SELECT groupid FROM pg_dist_local_group)) THEN 'coordinator (local)' @@ -583,19 +597,20 @@ BEGIN SELECT to_jsonb(database_properties.*) FROM ( SELECT datname, pa.rolname as database_owner, - pg_encoding_to_char(pd.encoding) as encoding, datlocprovider, - datistemplate, datallowconn, datconnlimit, - pt.spcname AS tablespace, datcollate, datctype, daticulocale, - datcollversion, datacl + pg_encoding_to_char(pd.encoding) as encoding, + datistemplate, datallowconn, datconnlimit, datacl, + pt.spcname AS tablespace, datcollate, datctype + %2$s -- >= pg15 options + %3$s -- >= pg16 options FROM pg_database pd JOIN pg_authid pa ON pd.datdba = pa.oid JOIN pg_tablespace pt ON pd.dattablespace = pt.oid - WHERE datname = '%s' + WHERE datname = '%1$s' ) database_properties ) AS database_properties, ( SELECT COUNT(*)=1 - FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%s') + FROM pg_dist_object WHERE objid = (SELECT oid FROM pg_database WHERE datname = '%1$s') ) AS pg_dist_object_record_for_db_exists, ( SELECT COUNT(*) > 0 @@ -604,7 +619,7 @@ BEGIN ) AS stale_pg_dist_object_record_for_a_db_exists ) q $$, - p_database_name, p_database_name + p_database_name, pg_ge_15_options, pg_ge_16_options ) ) q2 JOIN pg_dist_node USING (nodeid); From e56a1135726534ffa4a6038319fc2a1c77dd5b6b Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 08:09:09 +0300 Subject: [PATCH 174/180] Adds missing line --- src/test/regress/expected/multi_test_helpers.out | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index ea02a1cf830..70a541d2acd 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -555,6 +555,7 @@ BEGIN JOIN pg_dist_node USING (nodeid) ORDER BY node_type; END; +$func$ LANGUAGE plpgsql; -- For all nodes, returns database properties of given database, except -- oid, datfrozenxid and datminmxid. -- From 1cdae74a4cbf9a36c7fa2a8c3d76d583f0556b22 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 08:45:20 +0300 Subject: [PATCH 175/180] Fixes review notes --- src/backend/distributed/commands/database.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index b988122ef3c..7a3c29b418c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -614,47 +614,47 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", - quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); + quote_identifier(databaseForm->datallowconn ? "true" : "false")); appendStringInfo(&str, " IS_TEMPLATE = %s", - quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); + quote_identifier(databaseForm->datistemplate ? "true" : "false")); appendStringInfo(&str, " LC_COLLATE = %s", - quote_literal_cstr(collInfo.datcollate)); + quote_identifier(collInfo.datcollate)); - appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.datctype)); + appendStringInfo(&str, " LC_CTYPE = %s", quote_identifier(collInfo.datctype)); appendStringInfo(&str, " OWNER = %s", - quote_literal_cstr(GetUserNameFromId(databaseForm->datdba, false))); + quote_identifier(GetUserNameFromId(databaseForm->datdba, false))); appendStringInfo(&str, " TABLESPACE = %s", quote_identifier(GetTablespaceName(databaseForm->dattablespace))); appendStringInfo(&str, " ENCODING = %s", - quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); + quote_identifier(pg_encoding_to_char(databaseForm->encoding))); #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.datcollversion != NULL) { appendStringInfo(&str, " COLLATION_VERSION = %s", - quote_literal_cstr(collInfo.datcollversion)); + quote_identifier(collInfo.datcollversion)); } if (collInfo.daticulocale != NULL) { - appendStringInfo(&str, " ICU_LOCALE = %s", quote_literal_cstr( + appendStringInfo(&str, " ICU_LOCALE = %s", quote_identifier( collInfo.daticulocale)); } appendStringInfo(&str, " LOCALE_PROVIDER = %s", - quote_literal_cstr(GetLocaleProviderString( + quote_identifier(GetLocaleProviderString( databaseForm->datlocprovider))); #endif #if PG_VERSION_NUM >= PG_VERSION_16 if (collInfo.daticurules != NULL) { - appendStringInfo(&str, " ICU_RULES = %s", quote_literal_cstr( + appendStringInfo(&str, " ICU_RULES = %s", quote_identifier( collInfo.daticurules)); } #endif From c63f124c936b6a8257bfb708fbb54f74e174ebe8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 09:08:45 +0300 Subject: [PATCH 176/180] Fixes indentation --- src/backend/distributed/commands/database.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 7a3c29b418c..a515bc2a8b6 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -648,7 +648,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " LOCALE_PROVIDER = %s", quote_identifier(GetLocaleProviderString( - databaseForm->datlocprovider))); + databaseForm->datlocprovider))); #endif #if PG_VERSION_NUM >= PG_VERSION_16 From da72cd3cf17beb99671342d0d8a84f54b1afc522 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 10:03:02 +0300 Subject: [PATCH 177/180] Fixes quotes for db columns --- src/backend/distributed/commands/database.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index a515bc2a8b6..566ad7f983c 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -614,15 +614,15 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) appendStringInfo(&str, " CONNECTION LIMIT %d", databaseForm->datconnlimit); appendStringInfo(&str, " ALLOW_CONNECTIONS = %s", - quote_identifier(databaseForm->datallowconn ? "true" : "false")); + quote_literal_cstr(databaseForm->datallowconn ? "true" : "false")); appendStringInfo(&str, " IS_TEMPLATE = %s", - quote_identifier(databaseForm->datistemplate ? "true" : "false")); + quote_literal_cstr(databaseForm->datistemplate ? "true" : "false")); appendStringInfo(&str, " LC_COLLATE = %s", - quote_identifier(collInfo.datcollate)); + quote_literal_cstr(collInfo.datcollate)); - appendStringInfo(&str, " LC_CTYPE = %s", quote_identifier(collInfo.datctype)); + appendStringInfo(&str, " LC_CTYPE = %s", quote_literal_cstr(collInfo.datctype)); appendStringInfo(&str, " OWNER = %s", quote_identifier(GetUserNameFromId(databaseForm->datdba, false))); @@ -631,7 +631,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) quote_identifier(GetTablespaceName(databaseForm->dattablespace))); appendStringInfo(&str, " ENCODING = %s", - quote_identifier(pg_encoding_to_char(databaseForm->encoding))); + quote_literal_cstr(pg_encoding_to_char(databaseForm->encoding))); #if PG_VERSION_NUM >= PG_VERSION_15 if (collInfo.datcollversion != NULL) From 58bb165145fbe83a925b5a863432bd8a7a326203 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 16 Nov 2023 10:19:00 +0300 Subject: [PATCH 178/180] Removes unnecessary comments --- src/backend/distributed/commands/dependencies.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index c15f9fba4cb..e309ee86c7d 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -466,20 +466,6 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands); } - /*TODO: To reviewer: Having a code block for dependency makes sense */ - /* However dependency tree is based on pg metadata; which does not reflect */ - - /* actual database dependencies. I added this block just to point out the issue. - */ - - /* - * if(EnableCreateDatabasePropagation){ - * List *dbGrants = GrantOnDatabaseDDLCommands(dependency->objectId); - * databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants); - * - * } - */ - return databaseDDLCommands; } From 9241970e0ca3fd33b96f293c3a27c5427f9a49c9 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 20 Nov 2023 18:41:24 +0300 Subject: [PATCH 179/180] commit --- src/backend/distributed/commands/database.c | 252 +++++++----------- .../distributed/commands/dependencies.c | 33 ++- src/backend/distributed/commands/role.c | 2 +- .../distributed/deparser/citus_deparseutils.c | 9 - .../deparser/deparse_database_stmts.c | 78 ++---- src/backend/distributed/metadata/dependency.c | 1 - .../distributed/metadata/metadata_sync.c | 51 +--- src/include/distributed/commands.h | 4 +- src/include/distributed/deparser.h | 3 +- .../create_drop_database_propagation.out | 44 ++- src/test/regress/multi_1_schedule | 1 - .../sql/create_drop_database_propagation.sql | 45 ++-- 12 files changed, 224 insertions(+), 299 deletions(-) diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c index 566ad7f983c..ade604d1767 100644 --- a/src/backend/distributed/commands/database.c +++ b/src/backend/distributed/commands/database.c @@ -45,7 +45,7 @@ /* - * DatabaseCollationInfo is used to store collation related information of a database + * DatabaseCollationInfo is used to store collation related information of a database. */ typedef struct DatabaseCollationInfo { @@ -62,7 +62,6 @@ typedef struct DatabaseCollationInfo #endif } DatabaseCollationInfo; -static void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); static char * GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm); static DatabaseCollationInfo GetDatabaseCollation(Oid dbOid); @@ -71,10 +70,10 @@ static AlterOwnerStmt * RecreateAlterDatabaseOwnerStmt(Oid databaseOid); static char * GetLocaleProviderString(char datlocprovider); #endif static char * GetTablespaceName(Oid tablespaceOid); -static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, bool - missingOk); +static ObjectAddress * GetDatabaseAddressFromDatabaseName(char *databaseName, + bool missingOk); -static Oid get_database_owner(Oid db_oid); +static Oid get_database_owner(Oid dbId); /* controlled via GUC */ @@ -137,13 +136,13 @@ RecreateAlterDatabaseOwnerStmt(Oid databaseOid) * get_database_owner returns the Oid of the role owning the database */ static Oid -get_database_owner(Oid db_oid) +get_database_owner(Oid dbId) { - HeapTuple tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(db_oid)); + HeapTuple tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(dbId)); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_DATABASE), - errmsg("database with OID %u does not exist", db_oid))); + errmsg("database with OID %u does not exist", dbId))); } Oid dba = ((Form_pg_database) GETSTRUCT(tuple))->datdba; @@ -286,61 +285,13 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString, } -/* - * This function validates the options provided for the CREATE DATABASE command. - * It iterates over each option in the stmt->options list and checks if it's supported. - * If an unsupported option is found, or if a supported option has an invalid value, - * it raises an error. - * - * Parameters: - * stmt: A CreatedbStmt struct representing a CREATE DATABASE command. - * The options field is a list of DefElem structs, each representing an option. - * - * Currently, this function checks for the following: - * - The "oid" option is not supported. - * - The "template" option is only supported with the value "template1". - * - The "strategy" option is only supported with the value "wal_log". - * - * If any of these checks fail, the function calls ereport to raise an error. - */ -static void -EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt) -{ - DefElem *option = NULL; - foreach_ptr(option, stmt->options) - { - if (strcmp(option->defname, "oid") == 0) - { - ereport(ERROR, - errmsg("CREATE DATABASE option \"%s\" is not supported", - option->defname)); - } - - char *optionValue = defGetString(option); - - if (strcmp(option->defname, "template") == 0 && strcmp(optionValue, - "template1") != 0) - { - ereport(ERROR, errmsg("Only template1 is supported as template " - "parameter for CREATE DATABASE")); - } - - if (strcmp(option->defname, "strategy") == 0 && strcmp(optionValue, "wal_log") != - 0) - { - ereport(ERROR, errmsg("Only wal_log is supported as strategy " - "parameter for CREATE DATABASE")); - } - } -} - - /* * PostprocessAlterDatabaseStmt is executed before the statement is applied to the local - * postgres instance. + * Postgres instance. * - * In this stage, we can perform validations and prepare the commands that need to - * be run on all workers to create the database. + * In this stage, we perform validations that we want to ensure before delegating to + * previous utility hooks because it might not be convenient to throw an error in an + * implicit transaction that creates a database. */ List * PreprocessCreateDatabaseStmt(Node *node, const char *queryString, @@ -353,7 +304,6 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, EnsureCoordinator(); - /*validate the statement*/ CreatedbStmt *stmt = castNode(CreatedbStmt, node); EnsureSupportedCreateDatabaseCommand(stmt); @@ -363,7 +313,7 @@ PreprocessCreateDatabaseStmt(Node *node, const char *queryString, /* * PostprocessCreateDatabaseStmt is executed after the statement is applied to the local - * postgres instance. In this stage we can prepare the commands that need to be run on + * postgres instance. In this stage we prepare the commands that need to be run on * all workers to create the database. Since the CREATE DATABASE statement gives error * in a transaction block, we need to use NontransactionalNodeDDLTaskList to send the * CREATE DATABASE statement to the workers. @@ -379,6 +329,16 @@ PostprocessCreateDatabaseStmt(Node *node, const char *queryString) EnsureCoordinator(); + /* + * Given that CREATE DATABASE doesn't support "IF NOT EXISTS" and we're + * in the post-process, database must exist, hence missingOk = false. + */ + bool missingOk = false; + bool isPostProcess = true; + List *addresses = GetObjectAddressListFromParseTree(node, missingOk, + isPostProcess); + EnsureAllObjectDependenciesExistOnAllNodes(addresses); + char *createDatabaseCommand = DeparseTreeNode(node); List *commands = list_make3(DISABLE_DDL_PROPAGATION, @@ -435,20 +395,6 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString, } -/* - * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress - * of the database. - */ -static ObjectAddress * -GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) -{ - Oid databaseOid = get_database_oid(databaseName, missingOk); - ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); - ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, databaseOid); - return dbObjectAddress; -} - - /* * DropDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the * object of the DropdbStmt. @@ -477,6 +423,65 @@ CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess) } +/* + * EnsureSupportedCreateDatabaseCommand validates the options provided for the CREATE + * DATABASE command. + * + * Parameters: + * stmt: A CreatedbStmt struct representing a CREATE DATABASE command. + * The options field is a list of DefElem structs, each representing an option. + * + * Currently, this function checks for the following: + * - The "oid" option is not supported. + * - The "template" option is only supported with the value "template1". + * - The "strategy" option is only supported with the value "wal_log". + */ +void +EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt) +{ + DefElem *option = NULL; + foreach_ptr(option, stmt->options) + { + if (strcmp(option->defname, "oid") == 0) + { + ereport(ERROR, + errmsg("CREATE DATABASE option \"%s\" is not supported", + option->defname)); + } + + char *optionValue = defGetString(option); + + if (strcmp(option->defname, "template") == 0 && + strcmp(optionValue, "template1") != 0) + { + ereport(ERROR, errmsg("Only template1 is supported as template " + "parameter for CREATE DATABASE")); + } + + if (strcmp(option->defname, "strategy") == 0 && + strcmp(optionValue, "wal_log") != 0) + { + ereport(ERROR, errmsg("Only wal_log is supported as strategy " + "parameter for CREATE DATABASE")); + } + } +} + + +/* + * GetDatabaseAddressFromDatabaseName gets the database name and returns the ObjectAddress + * of the database. + */ +static ObjectAddress * +GetDatabaseAddressFromDatabaseName(char *databaseName, bool missingOk) +{ + Oid databaseOid = get_database_oid(databaseName, missingOk); + ObjectAddress *dbObjectAddress = palloc0(sizeof(ObjectAddress)); + ObjectAddressSet(*dbObjectAddress, DatabaseRelationId, databaseOid); + return dbObjectAddress; +} + + /* * GetTablespaceName gets the tablespace oid and returns the tablespace name. */ @@ -664,89 +669,34 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm) /* - * GrantOnDatabaseDDLCommands returns a list of sql statements to idempotently apply a - * GRANT on distributed databases. - */ -List * -GenerateGrantDatabaseCommandList(void) -{ - List *grantCommands = NIL; - - Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); - TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); - - HeapTuple tuple = NULL; - while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) - { - Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); - - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName( - NameStr(databaseForm->datname), false); - - /* skip databases that are not distributed */ - if (!IsAnyObjectDistributed(list_make1(dbAddress))) - { - continue; - } - - List *dbGrants = GrantOnDatabaseDDLCommands(databaseForm->oid); - - /* append dbGrants into grantCommands*/ - grantCommands = list_concat(grantCommands, dbGrants); - } - - heap_endscan(scan); - table_close(pgDatabaseRel, AccessShareLock); - - return grantCommands; -} - - -/* - * GenerateCreateDatabaseCommandList returns a list of CREATE DATABASE statements - * for all the databases. + * CreateDatabaseDDLCommand returns a CREATE DATABASE command to create given + * database * - * Commands in the list are wrapped by citus_internal_database_command() UDF - * to avoid from transaction block restrictions that apply to database commands + * Command is wrapped by citus_internal_database_command() UDF + * to avoid from transaction block restrictions that apply to database commands. */ -List * -GenerateCreateDatabaseCommandList(void) +char * +CreateDatabaseDDLCommand(Oid dbId) { - List *commands = NIL; - - Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock); - TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL); - - HeapTuple tuple = NULL; - while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + HeapTuple tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(dbId)); + if (!HeapTupleIsValid(tuple)) { - Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); - - ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName( - NameStr(databaseForm->datname), false); - - /* skip databases that are not distributed */ - if (!IsAnyObjectDistributed(list_make1(dbAddress))) - { - continue; - } + ereport(ERROR, (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database with OID %u does not exist", dbId))); + } - char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); + Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple); - StringInfo outerDbStmt = makeStringInfo(); + char *createStmt = GenerateCreateDatabaseStatementFromPgDatabase(databaseForm); - /* Generate the CREATE DATABASE statement */ - appendStringInfo(outerDbStmt, - "SELECT pg_catalog.citus_internal_database_command(%s)", - quote_literal_cstr( - createStmt)); + StringInfo outerDbStmt = makeStringInfo(); - /* Add the statement to the list of commands */ - commands = lappend(commands, outerDbStmt->data); - } + /* Generate the CREATE DATABASE statement */ + appendStringInfo(outerDbStmt, + "SELECT pg_catalog.citus_internal_database_command(%s)", + quote_literal_cstr(createStmt)); - heap_endscan(scan); - table_close(pgDatabaseRel, AccessShareLock); + ReleaseSysCache(tuple); - return commands; + return outerDbStmt->data; } diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index e309ee86c7d..6ce699cf5e4 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -457,16 +457,37 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) case OCLASS_DATABASE: { - List *databaseDDLCommands = NIL; + /* + * For the database where Citus is installed, only propagate the ownership of the + * database, only when the feature is on. + * + * This is because this database must exist on all nodes already so we shouldn't + * need to "CREATE" it on other nodes. However, we still need to correctly reflect + * its owner on other nodes too. + */ + if (dependency->objectId == MyDatabaseId && EnableAlterDatabaseOwner) + { + return DatabaseOwnerDDLCommands(dependency); + } - /* only propagate the ownership of the database when the feature is on */ - if (EnableAlterDatabaseOwner) + /* + * For the other databases, create the database on all nodes, only when the feature + * is on. + */ + if (dependency->objectId != MyDatabaseId && EnableCreateDatabasePropagation) { - List *ownerDDLCommands = DatabaseOwnerDDLCommands(dependency); - databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands); + char *databaseDDLCommand = CreateDatabaseDDLCommand(dependency->objectId); + + List *ddlCommands = list_make1(databaseDDLCommand); + + List *grantDDLCommands = GrantOnDatabaseDDLCommands(dependency->objectId); + + ddlCommands = list_concat(ddlCommands, grantDDLCommands); + + return ddlCommands; } - return databaseDDLCommands; + return NIL; } case OCLASS_PROC: diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index 407659aec3d..3177c73a0f4 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -1347,7 +1347,7 @@ CreateRoleStmtObjectAddress(Node *node, bool missing_ok, bool isPostprocess) * with the role the role needs to be visible on all connections used by the transaction, * meaning we can only use 1 connection per node. */ -void +static void EnsureSequentialModeForRoleDDL(void) { if (!IsTransactionBlock()) diff --git a/src/backend/distributed/deparser/citus_deparseutils.c b/src/backend/distributed/deparser/citus_deparseutils.c index 3f83f57fdfc..c0de4ae7c00 100644 --- a/src/backend/distributed/deparser/citus_deparseutils.c +++ b/src/backend/distributed/deparser/citus_deparseutils.c @@ -70,15 +70,6 @@ DefElemOptionToStatement(StringInfo buf, DefElem *option, break; } - #if PG_VERSION_NUM >= PG_VERSION_15 - case OPTION_FORMAT_OBJECT_ID: - { - Oid value = defGetObjectId(option); - appendStringInfo(buf, optionFormats[i].format, value); - break; - } - - #endif case OPTION_FORMAT_LITERAL_CSTR: { char *value = defGetString(option); diff --git a/src/backend/distributed/deparser/deparse_database_stmts.c b/src/backend/distributed/deparser/deparse_database_stmts.c index cd86eedf1a2..1ae45f920a6 100644 --- a/src/backend/distributed/deparser/deparse_database_stmts.c +++ b/src/backend/distributed/deparser/deparse_database_stmts.c @@ -13,17 +13,18 @@ #include "pg_version_compat.h" #include "catalog/namespace.h" +#include "commands/defrem.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" +#include "parser/parse_type.h" #include "utils/builtins.h" -#include "commands/defrem.h" #include "distributed/deparser.h" +#include "distributed/commands.h" #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/log_utils.h" -#include "parser/parse_type.h" static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt); @@ -49,26 +50,9 @@ const DefElemOptionFormat create_database_option_formats[] = { { "tablespace", " TABLESPACE %s", OPTION_FORMAT_STRING }, { "allow_connections", " ALLOW_CONNECTIONS %s", OPTION_FORMAT_BOOLEAN }, { "connection_limit", " CONNECTION LIMIT %d", OPTION_FORMAT_INTEGER }, - { "is_template", " IS_TEMPLATE %s", OPTION_FORMAT_BOOLEAN }, - { "oid", " OID %d", OPTION_FORMAT_OBJECT_ID } + { "is_template", " IS_TEMPLATE %s", OPTION_FORMAT_BOOLEAN } }; -/* - * DeparseAlterDatabaseOwnerStmt - * Deparse an AlterDatabaseOwnerStmt node - * - * This function is responsible for producing a string representation of an - * AlterDatabaseOwnerStmt node, which represents an ALTER DATABASE statement - * that changes the owner of a database. The output string includes the ALTER - * DATABASE keyword, the name of the database being altered, and the new owner - * of the database. - * - * Parameters: - * - node: a pointer to the AlterDatabaseOwnerStmt node to be deparsed - * - * Returns: - * - a string representation of the ALTER DATABASE statement - */ char * DeparseAlterDatabaseOwnerStmt(Node *node) { @@ -84,15 +68,6 @@ DeparseAlterDatabaseOwnerStmt(Node *node) } -/* - * - * AppendAlterDatabaseOwnerStmt - * Append an ALTER DATABASE statement for changing the owner of a database to the given StringInfo buffer. - * - * Parameters: - * - buf: The StringInfo buffer to append the statement to. - * - stmt: The AlterOwnerStmt representing the ALTER DATABASE statement to append. - */ static void AppendAlterDatabaseOwnerStmt(StringInfo buf, AlterOwnerStmt *stmt) { @@ -258,26 +233,25 @@ DeparseAlterDatabaseSetStmt(Node *node) static void AppendCreateDatabaseStmt(StringInfo buf, CreatedbStmt *stmt) { + /* + * Make sure that we don't try to deparse something that this + * function doesn't expect. + */ + EnsureSupportedCreateDatabaseCommand(stmt); + appendStringInfo(buf, "CREATE DATABASE %s", quote_identifier(stmt->dbname)); DefElem *option = NULL; - foreach_ptr(option, stmt->options) { - /*ValidateCreateDatabaseOptions(option); */ - DefElemOptionToStatement(buf, option, create_database_option_formats, lengthof(create_database_option_formats)); } } -/* - * Converts a CreatedbStmt structure into a SQL command string. - * Used in the deparsing of Create database statement. - */ char * DeparseCreateDatabaseStmt(Node *node) { @@ -300,20 +274,15 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) ifExistsStatement, quote_identifier(stmt->dbname)); - DefElem *option = NULL; - - - foreach_ptr(option, stmt->options) + if (list_length(stmt->options) > 1) { - /* if it is the first option then append with "WITH" else append with "," */ - if (option == linitial(stmt->options)) - { - appendStringInfo(buf, " WITH ( "); - } - else - { - appendStringInfo(buf, ", "); - } + /* FORCE is the only option that can be provided for this command */ + elog(ERROR, "got unexpected number of options for DROP DATABASE"); + } + else if (list_length(stmt->options) == 1) + { + DefElem *option = linitial(stmt->options); + appendStringInfo(buf, " WITH ( "); if (strcmp(option->defname, "force") == 0) { @@ -321,24 +290,17 @@ AppendDropDatabaseStmt(StringInfo buf, DropdbStmt *stmt) } else { + /* FORCE is the only option that can be provided for this command */ ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized DROP DATABASE option \"%s\"", option->defname))); } - /* if it is the last option then append with ")" */ - if (option == llast(stmt->options)) - { - appendStringInfo(buf, " )"); - } + appendStringInfo(buf, " )"); } } -/* - * Converts a DropdbStmt structure into a SQL command string. - * Used in the deparsing of drop database statement. - */ char * DeparseDropDatabaseStmt(Node *node) { diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index 989e957af36..be2ceb3e35f 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -698,7 +698,6 @@ SupportedDependencyByCitus(const ObjectAddress *address) case OCLASS_DATABASE: { - /* only to propagate its owner */ return true; } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index ce2201903fb..295fa3a368c 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -155,7 +155,6 @@ static char * RemoteSchemaIdExpressionByName(char *schemaName); static char * RemoteTypeIdExpression(Oid typeId); static char * RemoteCollationIdExpression(Oid colocationId); static char * RemoteTableIdExpression(Oid relationId); -static void SendDatabaseGrantSyncCommands(MetadataSyncContext *context); PG_FUNCTION_INFO_V1(start_metadata_sync_to_all_nodes); @@ -2049,6 +2048,10 @@ GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem) } +/* + * GrantOnDatabaseDDLCommands creates a list of ddl command for replicating the permissions + * of roles on databases. + */ List * GrantOnDatabaseDDLCommands(Oid databaseOid) { @@ -2079,6 +2082,10 @@ GrantOnDatabaseDDLCommands(Oid databaseOid) } +/* + * GenerateGrantOnDatabaseFromAclItem generates a query string for replicating a users permissions + * on a database. + */ List * GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem) { @@ -4658,13 +4665,6 @@ PropagateNodeWideObjectsCommandList(void) ddlCommands = list_concat(ddlCommands, alterRoleSetCommands); } - if (EnableCreateDatabasePropagation) - { - /* get commands for database creation */ - List *createDatabaseCommands = GenerateCreateDatabaseCommandList(); - ddlCommands = list_concat(ddlCommands, createDatabaseCommands); - } - return ddlCommands; } @@ -4736,13 +4736,6 @@ SyncDistributedObjects(MetadataSyncContext *context) * those tables. */ SendInterTableRelationshipCommands(context); - - /* - * After creation of databases and roles, send the grant database commands - * to the workers. - */ - - SendDatabaseGrantSyncCommands(context); } @@ -4768,34 +4761,6 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context) } -/* - * SendDatabaseGrantSyncCommands sends database grants to roles to workers with - * transactional or nontransactional mode according to transactionMode inside - * metadataSyncContext in case of EnableCreateDatabasePropagation GUC set. - * This function is called after SendNodeWideObjectsSyncCommands and SendDependencyCreationCommands - * because we need both databases and roles to be created on the worker. - * - */ -static void -SendDatabaseGrantSyncCommands(MetadataSyncContext *context) -{ - if (EnableCreateDatabasePropagation) - { - /* propagate node wide objects. It includes only roles for now. */ - List *commandList = GenerateGrantDatabaseCommandList(); - - if (commandList == NIL) - { - return; - } - - commandList = lcons(DISABLE_DDL_PROPAGATION, commandList); - commandList = lappend(commandList, ENABLE_DDL_PROPAGATION); - SendOrCollectCommandListToActivatedNodes(context, commandList); - } -} - - /* * SendShellTableDeletionCommands sends sequence, and shell table deletion * commands to workers with transactional or nontransactional mode according to diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 4391015cbbb..501fefeddab 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -243,8 +243,8 @@ extern List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk, bool isPostprocess); -extern List * GenerateCreateDatabaseCommandList(void); -extern List * GenerateGrantDatabaseCommandList(void); +extern void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt); +extern char * CreateDatabaseDDLCommand(Oid dbId); /* domain.c - forward declarations */ diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index f4bd6864ea7..f67e532384d 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -135,8 +135,7 @@ typedef enum OptionFormatType OPTION_FORMAT_STRING, OPTION_FORMAT_LITERAL_CSTR, OPTION_FORMAT_BOOLEAN, - OPTION_FORMAT_INTEGER, - OPTION_FORMAT_OBJECT_ID + OPTION_FORMAT_INTEGER } OptionFormatType; diff --git a/src/test/regress/expected/create_drop_database_propagation.out b/src/test/regress/expected/create_drop_database_propagation.out index bc3d5314f05..e0172f3e829 100644 --- a/src/test/regress/expected/create_drop_database_propagation.out +++ b/src/test/regress/expected/create_drop_database_propagation.out @@ -553,7 +553,7 @@ SELECT result from run_command_on_all_nodes( SELECT result from run_command_on_all_nodes( $$ - revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public + revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public $$ ) ORDER BY result; result @@ -572,7 +572,7 @@ select 1 from citus_remove_node('localhost', :worker_2_port); (1 row) CREATE DATABASE db_role_grants_test; -revoke connect,temp,temporary,create on database db_role_grants_test from public; +revoke connect,temp,temporary,create on database db_role_grants_test from public; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE ROLE%'; CREATE ROLE db_role_grants_test_role_missing_on_node_2; @@ -585,13 +585,13 @@ set citus.grep_remote_commands = '%GRANT%'; grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_exists_on_node_2; NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test_non_distributed TO db_role_grants_test_role_exists_on_node_2; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; NOTICE: issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test_non_distributed TO db_role_grants_test_role_missing_on_node_2; DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- check the privileges before add_node for database db_role_grants_test, @@ -894,7 +894,7 @@ SELECT result from run_command_on_all_nodes( t (3 rows) -grant connect,temp,temporary,create on database db_role_grants_test to public; +grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; SELECT result from run_command_on_all_nodes( $$ @@ -910,6 +910,40 @@ SELECT result from run_command_on_all_nodes( DROP ROLE db_role_grants_test_role_exists_on_node_2; DROP ROLE db_role_grants_test_role_missing_on_node_2; +select 1 from citus_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +set citus.enable_create_role_propagation TO off; +create role non_propagated_role; +NOTICE: not propagating CREATE ROLE/USER commands to other nodes +HINT: Connect to other nodes directly to manually create all necessary users and roles. +set citus.enable_create_role_propagation TO on; +set citus.enable_create_database_propagation TO on; +-- Make sure that we propagate non_propagated_role because it's a dependency of test_db. +-- And hence it becomes a distributed object. +create database test_db OWNER non_propagated_role; +create role propagated_role; +grant connect on database test_db to propagated_role; +SELECT 1 FROM citus_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT * FROM public.check_database_on_all_nodes('test_db') ORDER BY node_type; + node_type | result +--------------------------------------------------------------------- + coordinator (local) | {"database_properties": {"datacl": ["=Tc/non_propagated_role", "non_propagated_role=CTc/non_propagated_role", "propagated_role=c/non_propagated_role"], "datname": "test_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "non_propagated_role", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": ["=Tc/non_propagated_role", "non_propagated_role=CTc/non_propagated_role", "propagated_role=c/non_propagated_role"], "datname": "test_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "non_propagated_role", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": ["=Tc/non_propagated_role", "non_propagated_role=CTc/non_propagated_role", "propagated_role=c/non_propagated_role"], "datname": "test_db", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "non_propagated_role", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(3 rows) + +REVOKE CONNECT ON DATABASE test_db FROM propagated_role; +DROP DATABASE test_db; +DROP ROLE propagated_role, non_propagated_role; --clean up resources created by this test -- DROP TABLESPACE is not supported, so we need to drop it manually. SELECT result FROM run_command_on_all_nodes( diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 41672ae0756..9528cc70452 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -38,7 +38,6 @@ test: create_single_shard_table test: create_drop_database_propagation test: create_drop_database_propagation_pg15 test: create_drop_database_propagation_pg16 - # don't parallelize single_shard_table_udfs to make sure colocation ids are sequential test: single_shard_table_udfs test: schema_based_sharding diff --git a/src/test/regress/sql/create_drop_database_propagation.sql b/src/test/regress/sql/create_drop_database_propagation.sql index 82e179591db..c83548d6836 100644 --- a/src/test/regress/sql/create_drop_database_propagation.sql +++ b/src/test/regress/sql/create_drop_database_propagation.sql @@ -296,8 +296,6 @@ drop database distributed_db; set citus.enable_create_database_propagation TO off; drop database non_distributed_db; - - -- test role grants on DATABASE in metadata sync SELECT result from run_command_on_all_nodes( @@ -308,22 +306,19 @@ SELECT result from run_command_on_all_nodes( SELECT result from run_command_on_all_nodes( $$ - revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public + revoke connect,temp,temporary,create on database db_role_grants_test_non_distributed from public $$ ) ORDER BY result; SET citus.enable_create_database_propagation TO on; - - CREATE ROLE db_role_grants_test_role_exists_on_node_2; - select 1 from citus_remove_node('localhost', :worker_2_port); CREATE DATABASE db_role_grants_test; -revoke connect,temp,temporary,create on database db_role_grants_test from public; +revoke connect,temp,temporary,create on database db_role_grants_test from public; SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%CREATE ROLE%'; @@ -332,17 +327,13 @@ CREATE ROLE db_role_grants_test_role_missing_on_node_2; RESET citus.log_remote_commands ; RESET citus.grep_remote_commands; - - SET citus.log_remote_commands = true; set citus.grep_remote_commands = '%GRANT%'; grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2; -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; - - +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2; grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_exists_on_node_2; -grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; +grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test_non_distributed to db_role_grants_test_role_missing_on_node_2; -- check the privileges before add_node for database db_role_grants_test, -- role db_role_grants_test_role_exists_on_node_2 @@ -374,7 +365,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') @@ -416,7 +406,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') @@ -429,13 +418,11 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - RESET citus.log_remote_commands; RESET citus.grep_remote_commands; select 1 from citus_add_node('localhost', :worker_2_port); - -- check the privileges after add_node for database db_role_grants_test, -- role db_role_grants_test_role_exists_on_node_2 @@ -466,7 +453,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY') @@ -508,7 +494,6 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; - SELECT result from run_command_on_all_nodes( $$ select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test_non_distributed', 'TEMPORARY') @@ -521,7 +506,7 @@ SELECT result from run_command_on_all_nodes( $$ ) ORDER BY result; -grant connect,temp,temporary,create on database db_role_grants_test to public; +grant connect,temp,temporary,create on database db_role_grants_test to public; DROP DATABASE db_role_grants_test; @@ -533,8 +518,28 @@ SELECT result from run_command_on_all_nodes( DROP ROLE db_role_grants_test_role_exists_on_node_2; DROP ROLE db_role_grants_test_role_missing_on_node_2; +select 1 from citus_remove_node('localhost', :worker_2_port); + +set citus.enable_create_role_propagation TO off; +create role non_propagated_role; +set citus.enable_create_role_propagation TO on; + +set citus.enable_create_database_propagation TO on; + +-- Make sure that we propagate non_propagated_role because it's a dependency of test_db. +-- And hence it becomes a distributed object. +create database test_db OWNER non_propagated_role; + +create role propagated_role; +grant connect on database test_db to propagated_role; + +SELECT 1 FROM citus_add_node('localhost', :worker_2_port); +SELECT * FROM public.check_database_on_all_nodes('test_db') ORDER BY node_type; +REVOKE CONNECT ON DATABASE test_db FROM propagated_role; +DROP DATABASE test_db; +DROP ROLE propagated_role, non_propagated_role; --clean up resources created by this test From 72a9c22b6240ca9afc9759920679c65012f9decf Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Mon, 20 Nov 2023 19:52:19 +0300 Subject: [PATCH 180/180] style --- src/backend/distributed/metadata/metadata_sync.c | 4 ++-- src/backend/distributed/sql/citus--12.1-1--12.2-1.sql | 1 + .../distributed/sql/downgrades/citus--12.2-1--12.1-1.sql | 1 + .../sql/udfs/citus_internal_database_command/12.2-1.sql | 2 +- .../sql/udfs/citus_internal_database_command/latest.sql | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 295fa3a368c..f04768457e7 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -4017,7 +4017,7 @@ citus_internal_database_command(PG_FUNCTION_ARGS) GUC_ACTION_LOCAL, true, 0, false); /* - * createdb() uses ParseState to report the error position for the + * createdb() uses ParseState to report the error position for the * input command and the position is reported to be 0 when it's provided as NULL. * We're okay with that because we don't expect this UDF to be called with an incorrect * DDL command. @@ -4696,7 +4696,7 @@ SyncDistributedObjects(MetadataSyncContext *context) Assert(ShouldPropagate()); - /* send systemwide objects; i.e. roles and databases for now */ + /* Send systemwide objects, only roles for now */ SendNodeWideObjectsSyncCommands(context); /* diff --git a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql index 578a182eff4..63c4a527f8a 100644 --- a/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql +++ b/src/backend/distributed/sql/citus--12.1-1--12.2-1.sql @@ -1,4 +1,5 @@ -- citus--12.1-1--12.2-1 -- bump version to 12.2-1 + #include "udfs/citus_internal_database_command/12.2-1.sql" #include "udfs/citus_add_rebalance_strategy/12.2-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql b/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql index a8ca092c48c..d18f7257bc1 100644 --- a/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--12.2-1--12.1-1.sql @@ -1,4 +1,5 @@ -- citus--12.2-1--12.1-1 + DROP FUNCTION pg_catalog.citus_internal_database_command(text); #include "../udfs/citus_add_rebalance_strategy/10.1-1.sql" diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql index b20f6278e3f..9f6d873cc7d 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/12.2-1.sql @@ -1,5 +1,5 @@ -- --- citus_internal_database_command creates a database according to the given command. +-- citus_internal_database_command run given database command without transaction block restriction. CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) RETURNS void diff --git a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql index b20f6278e3f..9f6d873cc7d 100644 --- a/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_internal_database_command/latest.sql @@ -1,5 +1,5 @@ -- --- citus_internal_database_command creates a database according to the given command. +-- citus_internal_database_command run given database command without transaction block restriction. CREATE OR REPLACE FUNCTION pg_catalog.citus_internal_database_command(command text) RETURNS void