Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Configurable smoc_gin_ops index resolution #86

Merged
merged 1 commit into from
Oct 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ healpix_bare/healpix_bare.o : healpix_bare/healpix_bare.c

pg_version := $(word 2,$(shell $(PG_CONFIG) --version))
has_support_functions = $(if $(filter-out 9.% 10.% 11.%,$(pg_version)),y,n)
has_index_options = $(if $(filter-out 9.% 10.% 11.% 12.%,$(pg_version)),y,n)

crushtest: TESTS += $(CRUSH_TESTS)
crushtest: installcheck
Expand All @@ -108,6 +109,13 @@ PGS_SQL += pgs_gist_support.sql
TESTS += gist_support
endif

ifneq ($(USE_HEALPIX),0)
ifeq ($(has_index_options),y)
PGS_SQL += pgs_moc_options.sql
TESTS += moc_options
endif
endif

# "make test" uses a special initialization file that doesn't rely on "create extension"
test: pg_sphere.test.sql
$(pg_regress_installcheck) --temp-instance=tmp_check $(REGRESS_OPTS) init_test $(TESTS)
Expand Down Expand Up @@ -186,6 +194,11 @@ pg_sphere--1.3.0--1.3.1.sql:
ifeq ($(has_support_functions),y)
pg_sphere--1.3.1--1.3.2.sql: pgs_gist_support.sql.in
endif
ifneq ($(USE_HEALPIX),0)
vitcpp marked this conversation as resolved.
Show resolved Hide resolved
ifeq ($(has_index_options),y)
pg_sphere--1.3.1--1.3.2.sql: pgs_moc_options.sql.in
endif
endif
pg_sphere--1.3.1--1.3.2.sql: pgs_circle_sel.sql.in
vitcpp marked this conversation as resolved.
Show resolved Hide resolved
cat upgrade_scripts/[email protected] $^ > $@

Expand Down
21 changes: 15 additions & 6 deletions doc/indices.sgm
Original file line number Diff line number Diff line change
Expand Up @@ -121,23 +121,32 @@
The index works by casting all contained smocs to a fixed level, and
for each pixel at that level, storing which smocs overlap with that
pixel. This is especially beneficial for "overlaps" queries using
the <literal>&amp;&amp;</literal> operator. Two levels of granularity
are provided: the default opclass <literal>smoc_gin_ops</literal>
works on level 5 with a resolution of 12288 pixels, while the
opclass <literal>smoc_gin_ops_fine</literal> works on level 8 with
786432 pixels. The downside of that approach is that storing large
the <literal>&amp;&amp;</literal> operator.
The downside of that approach is that storing large
smocs like "all sky" (<literal>0/0-11</literal>) produces a large
number of index entries.
</para>
<para>
The default opclass <literal>smoc_gin_ops</literal> defaults to
working on level 5 with a resolution of 12288 pixels (12 * 4^5).
An alternative granularity can be selected by setting the
<literal>order</literal> parameter on the opclass (integer value
between 0 and 12; option only available on PG 13 and later).
The alternative <literal>smoc_gin_ops_fine</literal> opclass works
on level 8 with 786432 pixels.
</para>
<example>
<title>Index of smoc coverage objects</title>
<programlisting>
<![CDATA[CREATE TABLE ivoa (]]>
<![CDATA[ coverage smoc NOT NULL]]>
<![CDATA[);]]>
<![CDATA[-- Put in data now]]>
<![CDATA[-- Create index with the defaut smoc_gin_ops opclass (order 5)]]>
<![CDATA[CREATE INDEX ON ivoa USING GIN (coverage);]]>
<![CDATA[-- Alternative index with more detail]]>
<![CDATA[-- Alternative index with more detail on order 7]]>
<![CDATA[CREATE INDEX ivoa_order_7_idx ON ivoa USING GIN (coverage smoc_gin_ops (order = 7));]]>
<![CDATA[-- Alternative operator class with fixed order 8]]>
vitcpp marked this conversation as resolved.
Show resolved Hide resolved
<![CDATA[CREATE INDEX ivoa_fine_idx ON ivoa USING GIN (coverage smoc_gin_ops_fine);]]>
</programlisting>
</example>
Expand Down
40 changes: 40 additions & 0 deletions expected/moc_options.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
create table moc_opt (m smoc);
insert into moc_opt select format('9/%s', i)::smoc from generate_series(1, 1000) g(i);
analyze moc_opt;
create index moc_opt5 on moc_opt using gin (m);
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
QUERY PLAN
---------------------------------------------------------------
Bitmap Heap Scan on moc_opt (actual rows=1 loops=1)
Recheck Cond: (m && '9/1'::smoc)
Rows Removed by Index Recheck: 254
Heap Blocks: exact=4
-> Bitmap Index Scan on moc_opt5 (actual rows=255 loops=1)
Index Cond: (m && '9/1'::smoc)
(6 rows)

drop index moc_opt5;
create index moc_opt8 on moc_opt using gin (m smoc_gin_ops_fine);
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
QUERY PLAN
-------------------------------------------------------------
Bitmap Heap Scan on moc_opt (actual rows=1 loops=1)
Recheck Cond: (m && '9/1'::smoc)
Rows Removed by Index Recheck: 2
Heap Blocks: exact=1
-> Bitmap Index Scan on moc_opt8 (actual rows=3 loops=1)
Index Cond: (m && '9/1'::smoc)
(6 rows)

drop index moc_opt8;
create index moc_opt9 on moc_opt using gin (m smoc_gin_ops (order = 9));
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
QUERY PLAN
-------------------------------------------------------------
Bitmap Heap Scan on moc_opt (actual rows=1 loops=1)
Recheck Cond: (m && '9/1'::smoc)
Heap Blocks: exact=1
-> Bitmap Index Scan on moc_opt9 (actual rows=1 loops=1)
Index Cond: (m && '9/1'::smoc)
(5 rows)

1 change: 1 addition & 0 deletions pgs_moc_ops.sql.in
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,7 @@ CREATE OPERATOR CLASS smoc_gin_ops
FUNCTION 4 smoc_gin_consistent (internal, int2, smoc, int4, internal, internal, internal, internal),
--FUNCTION 5 smoc_gin_compare_partial (),
--FUNCTION 6 smoc_gin_tri_consistent (),
--FUNCTION 7 (smoc) smoc_gin_options (internal), -- needs PG13
STORAGE int4;

CREATE OPERATOR CLASS smoc_gin_ops_fine
Expand Down
12 changes: 12 additions & 0 deletions pgs_moc_options.sql.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
-- GIN opclass options

CREATE FUNCTION smoc_gin_options (internal)
RETURNS void
AS 'MODULE_PATHNAME'
LANGUAGE C
PARALLEL SAFE
IMMUTABLE
STRICT;

ALTER OPERATOR FAMILY smoc_gin_ops USING gin
ADD FUNCTION 7 (smoc) smoc_gin_options (internal);
14 changes: 14 additions & 0 deletions sql/moc_options.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
create table moc_opt (m smoc);
insert into moc_opt select format('9/%s', i)::smoc from generate_series(1, 1000) g(i);
analyze moc_opt;

create index moc_opt5 on moc_opt using gin (m);
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
drop index moc_opt5;

create index moc_opt8 on moc_opt using gin (m smoc_gin_ops_fine);
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
drop index moc_opt8;

create index moc_opt9 on moc_opt using gin (m smoc_gin_ops (order = 9));
explain (analyze, costs off, timing off, summary off) select * from moc_opt where m && '9/1';
29 changes: 25 additions & 4 deletions src/moc.c
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

#include <stddef.h>
#include <string.h>
#include <access/gin.h>
#include "access/gin.h"
#include "access/reloptions.h"

#include "circle.h"
#include "polygon.h"
Expand Down Expand Up @@ -45,6 +46,7 @@ PG_FUNCTION_INFO_V1(smoc_gin_extract_value_fine);
PG_FUNCTION_INFO_V1(smoc_gin_extract_query);
PG_FUNCTION_INFO_V1(smoc_gin_extract_query_fine);
PG_FUNCTION_INFO_V1(smoc_gin_consistent);
PG_FUNCTION_INFO_V1(smoc_gin_options);

int32 smoc_output_type = 0;

Expand Down Expand Up @@ -1079,7 +1081,6 @@ smoc_gin_extract_internal(Smoc *moc_a, int32 *nkeys, int gin_order)
if (*nkeys >= nalloc)
{
nalloc *= 2;
Assert(nalloc < 2000000);
keys = repalloc(keys, nalloc * sizeof(Datum));
}
keys[(*nkeys)++] = Int32GetDatum(p);
Expand All @@ -1094,8 +1095,9 @@ smoc_gin_extract_value(PG_FUNCTION_ARGS)
{
Smoc* moc_a = (Smoc *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
int32* nkeys = (int32 *) PG_GETARG_POINTER(1);
int order = SMOC_GIN_GET_ORDER();

PG_RETURN_DATUM(smoc_gin_extract_internal(moc_a, nkeys, MOC_GIN_ORDER));
PG_RETURN_DATUM(smoc_gin_extract_internal(moc_a, nkeys, order));
}

Datum
Expand All @@ -1114,13 +1116,14 @@ smoc_gin_extract_query(PG_FUNCTION_ARGS)
int32* nkeys = (int32 *) PG_GETARG_POINTER(1);
StrategyNumber st = PG_GETARG_UINT16(2);
int32* searchmode = (int32 *) PG_GETARG_POINTER(6);
int order = SMOC_GIN_GET_ORDER();

if (st == MOC_GIN_STRATEGY_SUBSET || (st == MOC_GIN_STRATEGY_EQUAL && moc_a->area == 0))
*searchmode = GIN_SEARCH_MODE_INCLUDE_EMPTY;
else if (st == MOC_GIN_STRATEGY_UNEQUAL)
*searchmode = GIN_SEARCH_MODE_ALL;

PG_RETURN_DATUM(smoc_gin_extract_internal(moc_a, nkeys, MOC_GIN_ORDER));
PG_RETURN_DATUM(smoc_gin_extract_internal(moc_a, nkeys, order));
}

Datum
Expand Down Expand Up @@ -1202,3 +1205,21 @@ smoc_gin_consistent(PG_FUNCTION_ARGS)
/* not reached */
PG_RETURN_NULL();
}

#if PG_VERSION_NUM >= 130000
Datum
smoc_gin_options(PG_FUNCTION_ARGS)
{
local_relopts *relopts = (local_relopts *) PG_GETARG_POINTER(0);

init_local_reloptions(relopts, sizeof(SMocGinOptions));
add_local_int_reloption(relopts, "order",
"smoc order to store in index",
MOC_GIN_ORDER_DEFAULT,
0,
12, /* maximum order fitting into 32bit */
offsetof(SMocGinOptions, order));

PG_RETURN_VOID();
}
#endif
19 changes: 18 additions & 1 deletion src/pgs_moc.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,12 +122,29 @@ next_interval(int32 a)

#define MOC_AREA_ALL_SKY 3458764513820540928

#define MOC_GIN_ORDER 5 /* order 5 has 12 * 4^5 = 12288 pixels */
#define MOC_GIN_ORDER_DEFAULT 5 /* order 5 has 12 * 4^5 = 12288 pixels */
#define MOC_GIN_ORDER_FINE 8 /* order 8 has 12 * 4^8 = 786432 pixels */
#define MOC_GIN_STRATEGY_INTERSECTS 1
#define MOC_GIN_STRATEGY_SUBSET 2
#define MOC_GIN_STRATEGY_SUPERSET 3
#define MOC_GIN_STRATEGY_EQUAL 4
#define MOC_GIN_STRATEGY_UNEQUAL 5

/* smoc_gin_ops opclass options */
#if PG_VERSION_NUM >= 130000
Datum smoc_gin_options(PG_FUNCTION_ARGS);

typedef struct
{
int32 vl_len_; /* varlena header (do not touch directly!) */
int order; /* smoc order to store in index (default 5) */
} SMocGinOptions;

#define SMOC_GIN_GET_ORDER() (PG_HAS_OPCLASS_OPTIONS() ? \
((SMocGinOptions *) PG_GET_OPCLASS_OPTIONS())->order : \
MOC_GIN_ORDER_DEFAULT)
#else
#define SMOC_GIN_GET_ORDER() MOC_GIN_ORDER_DEFAULT
#endif

#endif