Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PG16 - Add COPY FROM default tests #7143

Merged
merged 2 commits into from
Aug 24, 2023
+247 −0
Merged

PG16 - Add COPY FROM default tests #7143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
64ac072
Add COPY FROM default tests
naisila Aug 24, 2023
c7dae63
Address reviews
naisila Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 132 additions & 0 deletions src/test/regress/expected/pg16.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,138 @@ SELECT create_distributed_table('test_storage', 'a');
ALTER TABLE test_storage ALTER a SET STORAGE default;
ERROR: alter table command is currently unsupported
DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ENABLE|DISABLE|NO FORCE|FORCE ROW LEVEL SECURITY, ATTACH|DETACH PARTITION and TYPE subcommands are supported.
--
-- COPY FROM ... DEFAULT
-- Already supported in Citus, adding all PG tests with a distributed table
-- Relevant PG commit:
-- https://github.com/postgres/postgres/commit/9f8377f
CREATE TABLE copy_default (
id integer PRIMARY KEY,
text_value text NOT NULL DEFAULT 'test',
ts_value timestamp without time zone NOT NULL DEFAULT '2022-07-05'
);
SELECT create_distributed_table('copy_default', 'id');
create_distributed_table
---------------------------------------------------------------------

(1 row)

-- if DEFAULT is not specified, then the marker will be regular data
COPY copy_default FROM stdin;
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | D | Tue Jul 05 00:00:00 2022
(2 rows)

TRUNCATE copy_default;
COPY copy_default FROM stdin WITH (format csv);
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | \D | Tue Jul 05 00:00:00 2022
(2 rows)

TRUNCATE copy_default;
-- DEFAULT cannot be used in binary mode
COPY copy_default FROM stdin WITH (format binary, default '\D');
ERROR: cannot specify DEFAULT in BINARY mode
-- DEFAULT cannot be new line nor carriage return
COPY copy_default FROM stdin WITH (default E'\n');
ERROR: COPY default representation cannot use newline or carriage return
COPY copy_default FROM stdin WITH (default E'\r');
ERROR: COPY default representation cannot use newline or carriage return
-- DELIMITER cannot appear in DEFAULT spec
COPY copy_default FROM stdin WITH (delimiter ';', default 'test;test');
ERROR: COPY delimiter must not appear in the DEFAULT specification
-- CSV quote cannot appear in DEFAULT spec
COPY copy_default FROM stdin WITH (format csv, quote '"', default 'test"test');
ERROR: CSV quote character must not appear in the DEFAULT specification
-- NULL and DEFAULT spec must be different
COPY copy_default FROM stdin WITH (default '\N');
ERROR: NULL specification and DEFAULT specification cannot be the same
-- cannot use DEFAULT marker in column that has no DEFAULT value
COPY copy_default FROM stdin WITH (default '\D');
ERROR: unexpected default marker in COPY data
DETAIL: Column "id" has no default value.
CONTEXT: COPY copy_default, line 1: "\D value '2022-07-04'"
COPY copy_default FROM stdin WITH (format csv, default '\D');
ERROR: unexpected default marker in COPY data
DETAIL: Column "id" has no default value.
CONTEXT: COPY copy_default, line 1: "\D,value,2022-07-04"
-- The DEFAULT marker must be unquoted and unescaped or it's not recognized
COPY copy_default FROM stdin WITH (default '\D');
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | test | Mon Jul 04 00:00:00 2022
2 | \D | Mon Jul 04 00:00:00 2022
3 | "D" | Mon Jul 04 00:00:00 2022
(3 rows)

TRUNCATE copy_default;
COPY copy_default FROM stdin WITH (format csv, default '\D');
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | test | Mon Jul 04 00:00:00 2022
2 | \\D | Mon Jul 04 00:00:00 2022
3 | \D | Mon Jul 04 00:00:00 2022
(3 rows)

TRUNCATE copy_default;
-- successful usage of DEFAULT option in COPY
COPY copy_default FROM stdin WITH (default '\D');
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | test | Sun Jul 03 00:00:00 2022
3 | test | Tue Jul 05 00:00:00 2022
(3 rows)

TRUNCATE copy_default;
COPY copy_default FROM stdin WITH (format csv, default '\D');
SELECT * FROM copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | test | Sun Jul 03 00:00:00 2022
3 | test | Tue Jul 05 00:00:00 2022
(3 rows)

TRUNCATE copy_default;
\c - - - :worker_1_port
COPY pg16.copy_default FROM stdin WITH (format csv, default '\D');
SELECT * FROM pg16.copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | test | Sun Jul 03 00:00:00 2022
3 | test | Tue Jul 05 00:00:00 2022
(3 rows)

\c - - - :master_port
TRUNCATE pg16.copy_default;
\c - - - :worker_2_port
COPY pg16.copy_default FROM stdin WITH (format csv, default '\D');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as we sure that there are local shards here? if not, please consider adjusting the "id".

I'd be even fine to set citus.log_local_commands TO ON;

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are sure one of the two workers has shards, so I just connected to both of them.

SELECT * FROM pg16.copy_default ORDER BY id;
id | text_value | ts_value
---------------------------------------------------------------------
1 | value | Mon Jul 04 00:00:00 2022
2 | test | Sun Jul 03 00:00:00 2022
3 | test | Tue Jul 05 00:00:00 2022
(3 rows)

\c - - - :master_port
SET search_path TO pg16;
SET citus.shard_count TO 1;
SET citus.shard_replication_factor TO 1;
-- DEFAULT cannot be used in COPY TO
COPY (select 1 as test) TO stdout WITH (default '\D');
ERROR: COPY DEFAULT only available using COPY FROM
-- Tests for SQL/JSON: support the IS JSON predicate
-- Relevant PG commit:
-- https://github.com/postgres/postgres/commit/6ee30209
Expand Down
115 changes: 115 additions & 0 deletions src/test/regress/sql/pg16.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,121 @@ SELECT result FROM run_command_on_all_nodes
SELECT create_distributed_table('test_storage', 'a');
ALTER TABLE test_storage ALTER a SET STORAGE default;

--
-- COPY FROM ... DEFAULT
-- Already supported in Citus, adding all PG tests with a distributed table
-- Relevant PG commit:
-- https://github.com/postgres/postgres/commit/9f8377f
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy on the shards that are local goes through a different codepath. So, I think it'd be very useful to add one very basic test to make sure it works.

A good alternative could be to create a citus-local (or a reference table), where the shard will be local to the coodinator.

Or, connect to one of the workers and ensure the copy is happening at least one local shard.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or, connect to one of the workers and ensure the copy is happening at least one local shard.

Thanks, I picked this one.

CREATE TABLE copy_default (
id integer PRIMARY KEY,
text_value text NOT NULL DEFAULT 'test',
ts_value timestamp without time zone NOT NULL DEFAULT '2022-07-05'
);
SELECT create_distributed_table('copy_default', 'id');

-- if DEFAULT is not specified, then the marker will be regular data
COPY copy_default FROM stdin;
1 value '2022-07-04'
2 \D '2022-07-05'
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

COPY copy_default FROM stdin WITH (format csv);
1,value,2022-07-04
2,\D,2022-07-05
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

-- DEFAULT cannot be used in binary mode
COPY copy_default FROM stdin WITH (format binary, default '\D');

-- DEFAULT cannot be new line nor carriage return
COPY copy_default FROM stdin WITH (default E'\n');
COPY copy_default FROM stdin WITH (default E'\r');

-- DELIMITER cannot appear in DEFAULT spec
COPY copy_default FROM stdin WITH (delimiter ';', default 'test;test');

-- CSV quote cannot appear in DEFAULT spec
COPY copy_default FROM stdin WITH (format csv, quote '"', default 'test"test');

-- NULL and DEFAULT spec must be different
COPY copy_default FROM stdin WITH (default '\N');

-- cannot use DEFAULT marker in column that has no DEFAULT value
COPY copy_default FROM stdin WITH (default '\D');
\D value '2022-07-04'
2 \D '2022-07-05'
\.

COPY copy_default FROM stdin WITH (format csv, default '\D');
\D,value,2022-07-04
2,\D,2022-07-05
\.

-- The DEFAULT marker must be unquoted and unescaped or it's not recognized
COPY copy_default FROM stdin WITH (default '\D');
1 \D '2022-07-04'
2 \\D '2022-07-04'
3 "\D" '2022-07-04'
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

COPY copy_default FROM stdin WITH (format csv, default '\D');
1,\D,2022-07-04
2,\\D,2022-07-04
3,"\D",2022-07-04
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

-- successful usage of DEFAULT option in COPY
COPY copy_default FROM stdin WITH (default '\D');
1 value '2022-07-04'
2 \D '2022-07-03'
3 \D \D
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

COPY copy_default FROM stdin WITH (format csv, default '\D');
1,value,2022-07-04
2,\D,2022-07-03
3,\D,\D
\.
SELECT * FROM copy_default ORDER BY id;
TRUNCATE copy_default;

\c - - - :worker_1_port
COPY pg16.copy_default FROM stdin WITH (format csv, default '\D');
1,value,2022-07-04
2,\D,2022-07-03
3,\D,\D
\.
SELECT * FROM pg16.copy_default ORDER BY id;

\c - - - :master_port
TRUNCATE pg16.copy_default;

\c - - - :worker_2_port
COPY pg16.copy_default FROM stdin WITH (format csv, default '\D');
1,value,2022-07-04
2,\D,2022-07-03
3,\D,\D
\.
SELECT * FROM pg16.copy_default ORDER BY id;

\c - - - :master_port
SET search_path TO pg16;
SET citus.shard_count TO 1;
SET citus.shard_replication_factor TO 1;

-- DEFAULT cannot be used in COPY TO
COPY (select 1 as test) TO stdout WITH (default '\D');

-- Tests for SQL/JSON: support the IS JSON predicate
-- Relevant PG commit:
-- https://github.com/postgres/postgres/commit/6ee30209
Expand Down