Skip to content

Commit

Permalink
Add auto setup pg package
Browse files Browse the repository at this point in the history
  • Loading branch information
tonyhb committed Sep 16, 2024
1 parent e7ee5e7 commit dbbd684
Show file tree
Hide file tree
Showing 8 changed files with 317 additions and 49 deletions.
49 changes: 12 additions & 37 deletions internal/test/pg_init.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"
"testing"

"github.com/inngest/dbcap/pkg/replicator/pg/pgsetup"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -50,16 +51,17 @@ func StartPG(t *testing.T, ctx context.Context, opts StartPGOpts) (tc.Container,
require.NoError(t, err)
}

if !opts.DisableCreateRoles {
// Create the replication slot.
err := prepareRoles(ctx, conn)
require.NoError(t, err)
}
if !opts.DisableCreateSlot {
// Create the replication slot.
err := createReplicationSlot(ctx, conn)
require.NoError(t, err)
}
connCfg, err := pgx.ParseConfig(connString(t, c))
require.NoError(t, err, "Failed to parse config")

sr, err := pgsetup.Setup(ctx, pgsetup.SetupOpts{
AdminConfig: *connCfg,
Password: "password",
DisableCreateUser: opts.DisableCreateRoles,
DisableCreateRoles: opts.DisableCreateRoles,
DisableCreateSlot: opts.DisableCreateSlot,
})
require.NoError(t, err, "Setup results: %#v", sr.Results())

err = createTables(ctx, conn)
require.NoError(t, err)
Expand Down Expand Up @@ -98,33 +100,6 @@ func connOpts(t *testing.T, c tc.Container) pgx.ConnConfig {
return *cfg
}

func prepareRoles(ctx context.Context, c *pgconn.PgConn) error {
stmt := `
CREATE USER inngest WITH REPLICATION PASSWORD 'password';
GRANT USAGE ON SCHEMA public TO inngest;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO inngest;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO inngest;
CREATE PUBLICATION inngest FOR ALL TABLES;
`
res := c.Exec(ctx, stmt)
if err := res.Close(); err != nil {
return err
}
return nil
}

func createReplicationSlot(ctx context.Context, c *pgconn.PgConn) error {
stmt := `
-- pgoutput logical repl plugin
SELECT pg_create_logical_replication_slot('inngest_cdc', 'pgoutput');
`
res := c.Exec(ctx, stmt)
if err := res.Close(); err != nil {
return err
}
return nil
}

func createTables(ctx context.Context, c *pgconn.PgConn) error {
stmt := `
CREATE EXTENSION IF NOT EXISTS "pgcrypto" WITH SCHEMA public;
Expand Down
1 change: 1 addition & 0 deletions pkg/consts/pgconsts/pgconsts.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package pgconsts

const (
Username = "inngest"
SlotName = "inngest_cdc"
PublicationName = "inngest"
)
17 changes: 11 additions & 6 deletions pkg/replicator/pg.go → pkg/replicator/pg/pg.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package replicator
package pg

import (
"context"
Expand All @@ -14,6 +14,7 @@ import (
"github.com/inngest/dbcap/pkg/changeset"
"github.com/inngest/dbcap/pkg/consts/pgconsts"
"github.com/inngest/dbcap/pkg/decoder"
"github.com/inngest/dbcap/pkg/replicator"
"github.com/inngest/dbcap/pkg/schema"
"github.com/jackc/pglogrepl"
"github.com/jackc/pgx/v5"
Expand All @@ -25,6 +26,10 @@ var (
ReadTimeout = time.Second * 5
CommitInterval = time.Second * 5

ErrInvalidCredentials = fmt.Errorf("TODO")

ErrConnectionTimeout = fmt.Errorf("TODO")

ErrLogicalReplicationNotSetUp = fmt.Errorf("ERR_PG_001: Your database does not have logical replication configured. You must set the WAL level to 'logical' to stream events.")

ErrReplicationSlotNotFound = fmt.Errorf("ERR_PG_002: The replication slot 'inngest_cdc' doesn't exist in your database. Please create the logical replication slot to stream events.")
Expand All @@ -34,7 +39,7 @@ var (

// PostgresReplicator is a Replicator with added postgres functionality.
type PostgresReplicator interface {
Replicator
replicator.Replicator

// ReplicationSlot returns the replication slot data or an error.
//
Expand All @@ -53,11 +58,11 @@ type PostgresOpts struct {
Config pgx.ConnConfig
// WatermarkSaver saves the current watermark to local storage. This should be paired with a
// WatermarkLoader to load offsets when the replicator restarts.
WatermarkSaver WatermarkSaver
WatermarkSaver replicator.WatermarkSaver
// WatermarkLoader, if specified, loads watermarks for the given connection to start replication
// from a given offset. If this isn't specified, replication will start from the latest point in
// the Postgres server's WAL.
WatermarkLoader WatermarkLoader
WatermarkLoader replicator.WatermarkLoader
// Log, if specified, is the stdlib logger used to log debug and warning messages during
// replication.
Log *slog.Logger
Expand Down Expand Up @@ -140,9 +145,9 @@ func (p *pg) Close(ctx context.Context) error {
return nil
}

func (p *pg) TestConnection(ctx context.Context) error {
func (p *pg) TestConnection(ctx context.Context) (replicator.ConnectionResult, error) {
_, err := p.ReplicationSlot(ctx)
return err
return nil, err
}

func (p *pg) ReplicationSlot(ctx context.Context) (ReplicationSlot, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/replicator/pg_test.go → pkg/replicator/pg/pg_test.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package replicator
package pg

import (
"context"
Expand Down
Loading

0 comments on commit dbbd684

Please sign in to comment.