From 398f85b85b228f1a21bc4c72b8e755f8a4c0b7b1 Mon Sep 17 00:00:00 2001 From: Sean McGary Date: Fri, 6 Sep 2024 13:38:51 -0500 Subject: [PATCH] Setup sqlite with migrations --- .../migrations/202409061249_bootstrapDb/up.go | 97 ++++++++++++++++ internal/sqlite/sqlite.go | 18 ++- .../storage/sqlite/migrations/migrator.go | 108 ++++++++++++++++++ internal/storage/sqlite/sqlite_test.go | 83 ++++++++++++++ internal/storage/storage.go | 2 - internal/tests/utils.go | 11 ++ scripts/generateSqliteMigration.sh | 38 ++++++ 7 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 internal/sqlite/migrations/202409061249_bootstrapDb/up.go create mode 100644 internal/storage/sqlite/migrations/migrator.go create mode 100644 internal/storage/sqlite/sqlite_test.go create mode 100755 scripts/generateSqliteMigration.sh diff --git a/internal/sqlite/migrations/202409061249_bootstrapDb/up.go b/internal/sqlite/migrations/202409061249_bootstrapDb/up.go new file mode 100644 index 00000000..09ac2d31 --- /dev/null +++ b/internal/sqlite/migrations/202409061249_bootstrapDb/up.go @@ -0,0 +1,97 @@ +package _202409061249_bootstrapDb + +import ( + "fmt" + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + queries := []string{ + `CREATE TABLE IF NOT EXISTS blocks ( + number INTEGER NOT NULL PRIMARY KEY, + hash text NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + block_time DATETIME NOT NULL, + updated_at DATETIME DEFAULT NULL, + deleted_at DATETIME DEFAULT NULL + )`, + `CREATE TABLE IF NOT EXISTS transactions ( + block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE, + transaction_hash TEXT NOT NULL, + transaction_index INTEGER NOT NULL, + from_address TEXT NOT NULL, + to_address TEXT DEFAULT NULL, + contract_address TEXT DEFAULT NULL, + bytecode_hash TEXT DEFAULT NULL, + gas_used INTEGER DEFAULT NULL, + cumulative_gas_used INTEGER DEFAULT NULL, + effective_gas_price INTEGER DEFAULT NULL, + created_at DATETIME DEFAULT current_timestamp, + updated_at DATETIME DEFAULT NULL, + deleted_at DATETIME DEFAULT NULL, + UNIQUE(block_number, transaction_hash, transaction_index) + )`, + `CREATE TABLE IF NOT EXISTS transaction_logs ( + transaction_hash TEXT NOT NULL REFERENCES transactions(transaction_hash) ON DELETE CASCADE, + address TEXT NOT NULL, + arguments TEXT, + event_name TEXT NOT NULL, + log_index INTEGER NOT NULL, + block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE, + transaction_index INTEGER NOT NULL, + output_data TEXT + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME zone, + deleted_at DATETIME zone, + UNIQUE(transaction_hash, log_index) + )`, + `CREATE TABLE IF NOT EXISTS contracts ( + contract_address TEXT NOT NULL, + contract_abi TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME, + deleted_at DATETIME, + bytecode_hash TEXT DEFAULT NULL, + verified INTEGER DEFAULT false, + matching_contract_address TEXT DEFAULT NULL, + checked_for_proxy INTEGER DEFAULT 0 NOT NULL, + checked_for_abi INTEGER NOT NULL, + UNIQUE(contract_address) + )`, + `CREATE TABLE IF NOT EXISTS proxy_contracts ( + block_number INTEGER NOT NULL, + contract_address TEXT NOT NULL PRIMARY KEY REFERENCES contracts(contract_address) ON DELETE CASCADE, + proxy_contract_address TEXT NOT NULL REFERENCES contracts(contract_address) ON DELETE CASCADE, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at DATETIME, + deleted_at DATETIME + )`, + `CREATE TABLE IF NOT EXISTS operator_restaked_strategies ( + block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE, + operator TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME, + deleted_at DATETIME, + block_time DATETIME NOT NULL, + avs_directory_address TEXT + );`, + } + + for _, query := range queries { + res := grm.Exec(query) + if res.Error != nil { + fmt.Printf("Failed to run migration query: %s - %+v\n", query, res.Error) + return res.Error + } + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409061249_bootstrapDb" +} diff --git a/internal/sqlite/sqlite.go b/internal/sqlite/sqlite.go index 4837f601..8ad06264 100644 --- a/internal/sqlite/sqlite.go +++ b/internal/sqlite/sqlite.go @@ -11,5 +11,21 @@ func NewSqlite(path string) gorm.Dialector { } func NewGormSqliteFromSqlite(sqlite gorm.Dialector) (*gorm.DB, error) { - return gorm.Open(sqlite, &gorm.Config{}) + db, err := gorm.Open(sqlite, &gorm.Config{}) + if err != nil { + return nil, err + } + + pragmas := []string{ + `PRAGMA foreign_keys = ON;`, + `PRAGMA journal_mode = WAL;`, + } + + for _, pragma := range pragmas { + res := db.Exec(pragma) + if res.Error != nil { + return nil, res.Error + } + } + return db, nil } diff --git a/internal/storage/sqlite/migrations/migrator.go b/internal/storage/sqlite/migrations/migrator.go new file mode 100644 index 00000000..366e9f33 --- /dev/null +++ b/internal/storage/sqlite/migrations/migrator.go @@ -0,0 +1,108 @@ +package migrations + +import ( + "database/sql" + "fmt" + _202409061249_bootstrapDb "github.com/Layr-Labs/sidecar/internal/sqlite/migrations/202409061249_bootstrapDb" + "go.uber.org/zap" + "gorm.io/gorm" + "time" +) + +type ISqliteMigration interface { + Up(grm *gorm.DB) error + GetName() string +} + +type SqliteMigrator struct { + Db *sql.DB + GDb *gorm.DB + Logger *zap.Logger +} + +func NewSqliteMigrator(gDb *gorm.DB, l *zap.Logger) *SqliteMigrator { + return &SqliteMigrator{ + GDb: gDb, + Logger: l, + } +} + +func (m *SqliteMigrator) MigrateAll() error { + err := m.CreateMigrationTablesIfNotExists() + if err != nil { + return err + } + + migrations := []ISqliteMigration{ + &_202409061249_bootstrapDb.SqliteMigration{}, + } + + for _, migration := range migrations { + err := m.Migrate(migration) + if err != nil { + panic(err) + } + } + return nil +} + +func (m *SqliteMigrator) CreateMigrationTablesIfNotExists() error { + queries := []string{ + `CREATE TABLE IF NOT EXISTS migrations ( + name TEXT PRIMARY KEY, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT NULL, + deleted_at DATETIME DEFAULT NULL + )`, + } + + for _, query := range queries { + res := m.GDb.Exec(query) + if res.Error != nil { + m.Logger.Sugar().Errorw("Failed to create migration table", zap.Error(res.Error)) + return res.Error + } + } + return nil +} + +func (m *SqliteMigrator) Migrate(migration ISqliteMigration) error { + name := migration.GetName() + + // find migration by name + var migrationRecord Migrations + result := m.GDb.Find(&migrationRecord, "name = ?", name).Limit(1) + + if result.Error == nil && result.RowsAffected == 0 { + m.Logger.Sugar().Infof("Running migration '%s'", name) + // run migration + err := migration.Up(m.GDb) + if err != nil { + m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to run migration '%s'", name), zap.Error(err)) + return err + } + + // record migration + migrationRecord = Migrations{ + Name: name, + } + result = m.GDb.Create(&migrationRecord) + if result.Error != nil { + m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to record migration '%s'", name), zap.Error(result.Error)) + return result.Error + } + } else if result.Error != nil { + m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to find migration '%s'", name), zap.Error(result.Error)) + return result.Error + } else if result.RowsAffected > 0 { + m.Logger.Sugar().Infof("Migration %s already run", name) + return nil + } + return nil +} + +type Migrations struct { + Name string `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/internal/storage/sqlite/sqlite_test.go b/internal/storage/sqlite/sqlite_test.go new file mode 100644 index 00000000..2df00d11 --- /dev/null +++ b/internal/storage/sqlite/sqlite_test.go @@ -0,0 +1,83 @@ +package sqlite + +import ( + "fmt" + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/internal/logger" + "github.com/Layr-Labs/sidecar/internal/storage" + "github.com/Layr-Labs/sidecar/internal/storage/sqlite/migrations" + "github.com/Layr-Labs/sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" + "time" +) + +func setup() (*gorm.DB, *zap.Logger, *config.Config) { + cfg := config.NewConfig() + l, err := logger.NewLogger(&logger.LoggerConfig{Debug: true}) + db, err := tests.GetSqliteDatabaseConnection() + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + return db, l, cfg +} + +func teardown(db *gorm.DB, l *zap.Logger) { + queries := []string{ + `truncate table blocks cascade`, + `truncate table transactions cascade`, + `truncate table transaction_logs cascade`, + `truncate table transaction_logs cascade`, + } + for _, query := range queries { + res := db.Exec(query) + if res.Error != nil { + l.Sugar().Errorw("Failed to truncate table", "error", res.Error) + } + } +} + +func Test_SqliteBlockstore(t *testing.T) { + t.Run("Blocks", func(t *testing.T) { + db, l, cfg := setup() + + sqliteStore := NewSqliteBlockStore(db, l, cfg) + + t.Run("InsertBlockAtHeight", func(t *testing.T) { + block := &storage.Block{ + Number: 100, + Hash: "some hash", + BlockTime: time.Now(), + } + + insertedBlock, err := sqliteStore.InsertBlockAtHeight(block.Number, block.Hash, uint64(block.BlockTime.Unix())) + if err != nil { + t.Errorf("Failed to insert block: %v", err) + } + assert.NotNil(t, insertedBlock) + assert.Equal(t, block.Number, insertedBlock.Number) + assert.Equal(t, block.Hash, insertedBlock.Hash) + }) + t.Run("Fail to insert a duplicate block", func(t *testing.T) { + block := &storage.Block{ + Number: 100, + Hash: "some hash", + BlockTime: time.Now(), + } + + _, err := sqliteStore.InsertBlockAtHeight(block.Number, block.Hash, uint64(block.BlockTime.Unix())) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "UNIQUE constraint failed") + fmt.Printf("Error: %v\n", err) + }) + t.Run("InsertBlockTransaction", func(t *testing.T) { + + }) + }) +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 6f317a35..0ce24f51 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -19,11 +19,9 @@ type BlockStore interface { // Tables type Block struct { - Id uint64 `gorm:"type:serial"` Number uint64 Hash string BlockTime time.Time - BlobPath string CreatedAt time.Time UpdatedAt time.Time DeletedAt time.Time diff --git a/internal/tests/utils.go b/internal/tests/utils.go index 14dfe528..f0a0d1d5 100644 --- a/internal/tests/utils.go +++ b/internal/tests/utils.go @@ -3,6 +3,7 @@ package tests import ( "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/internal/postgres" + sqlite2 "github.com/Layr-Labs/sidecar/internal/sqlite" "gorm.io/gorm" ) @@ -28,3 +29,13 @@ func GetDatabaseConnection(cfg *config.Config) (*postgres.Postgres, *gorm.DB, er } return db, grm, nil } + +const sqliteInMemoryPath = "file::memory:?cache=shared" + +func GetSqliteDatabaseConnection() (*gorm.DB, error) { + db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqliteInMemoryPath)) + if err != nil { + panic(err) + } + return db, nil +} diff --git a/scripts/generateSqliteMigration.sh b/scripts/generateSqliteMigration.sh new file mode 100755 index 00000000..33a3061c --- /dev/null +++ b/scripts/generateSqliteMigration.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +name=$1 + +if [[ -z $name ]]; then + echo "Usage: $0 " + exit 1 +fi + +timestamp=$(date +"%Y%m%d%H%M") + +migration_name="${timestamp}_${name}" + +migrations_dir="./internal/sqlite/migrations/${migration_name}" +migration_file="${migrations_dir}/up.go" + +mkdir -p $migrations_dir || true + +# heredoc that creates a migration go file with an Up function +cat > $migration_file <