Setup sqlite with migrations

internal/sqlite/migrations/202409061249_bootstrapDb/up.go

@@ -0,0 +1,97 @@
+package _202409061249_bootstrapDb
+
+import (
+	"fmt"
+	"gorm.io/gorm"
+)
+
+type SqliteMigration struct {
+}
+
+func (m *SqliteMigration) Up(grm *gorm.DB) error {
+	queries := []string{
+		`CREATE TABLE IF NOT EXISTS blocks (
+			number INTEGER NOT NULL PRIMARY KEY,
+			hash text NOT NULL,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+			block_time DATETIME NOT NULL,
+			updated_at DATETIME DEFAULT NULL,
+			deleted_at DATETIME DEFAULT NULL
+    	)`,
+		`CREATE TABLE IF NOT EXISTS transactions (
+			block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE,
+			transaction_hash TEXT NOT NULL,
+			transaction_index INTEGER NOT NULL,
+			from_address TEXT NOT NULL,
+			to_address TEXT DEFAULT NULL,
+			contract_address TEXT DEFAULT NULL,
+			bytecode_hash TEXT DEFAULT NULL,
+			gas_used INTEGER DEFAULT NULL,
+			cumulative_gas_used INTEGER DEFAULT NULL,
+			effective_gas_price INTEGER DEFAULT NULL,
+			created_at DATETIME DEFAULT current_timestamp,
+			updated_at DATETIME DEFAULT NULL,
+			deleted_at DATETIME DEFAULT NULL,
+			UNIQUE(block_number, transaction_hash, transaction_index)
+		)`,
+		`CREATE TABLE IF NOT EXISTS transaction_logs (
+			transaction_hash TEXT NOT NULL REFERENCES transactions(transaction_hash) ON DELETE CASCADE,
+			address TEXT NOT NULL,
+			arguments TEXT,
+			event_name TEXT NOT NULL,
+			log_index INTEGER NOT NULL,
+			block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE,
+			transaction_index INTEGER NOT NULL,
+			output_data TEXT
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+			updated_at DATETIME zone,
+			deleted_at DATETIME zone,
+			UNIQUE(transaction_hash, log_index)
+    	)`,
+		`CREATE TABLE IF NOT EXISTS contracts (
+			contract_address TEXT NOT NULL,
+			contract_abi TEXT,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+			updated_at DATETIME,
+			deleted_at DATETIME,
+			bytecode_hash TEXT DEFAULT NULL,
+			verified INTEGER DEFAULT false,
+			matching_contract_address TEXT DEFAULT NULL,
+			checked_for_proxy INTEGER DEFAULT 0 NOT NULL,
+			checked_for_abi INTEGER NOT NULL,
+			UNIQUE(contract_address)
+		)`,
+		`CREATE TABLE IF NOT EXISTS proxy_contracts (
+			block_number INTEGER NOT NULL,
+			contract_address TEXT NOT NULL PRIMARY KEY REFERENCES contracts(contract_address) ON DELETE CASCADE,
+			proxy_contract_address TEXT NOT NULL REFERENCES contracts(contract_address) ON DELETE CASCADE,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
+			updated_at DATETIME,
+			deleted_at DATETIME
+		)`,
+		`CREATE TABLE IF NOT EXISTS operator_restaked_strategies (
+			block_number INTEGER NOT NULL REFERENCES blocks(number) ON DELETE CASCADE,
+			operator TEXT NOT NULL,
+			avs TEXT NOT NULL,
+			strategy TEXT NOT NULL,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+			updated_at DATETIME,
+			deleted_at DATETIME,
+			block_time DATETIME NOT NULL,
+			avs_directory_address TEXT
+		);`,
+	}
+
+	for _, query := range queries {
+		res := grm.Exec(query)
+		if res.Error != nil {
+			fmt.Printf("Failed to run migration query: %s - %+v\n", query, res.Error)
+			return res.Error
+		}
+	}
+	return nil
+}
+
+func (m *SqliteMigration) GetName() string {
+	return "202409061249_bootstrapDb"
+}

internal/sqlite/sqlite.go

@@ -11,5 +11,21 @@ func NewSqlite(path string) gorm.Dialector {
 }
 
 func NewGormSqliteFromSqlite(sqlite gorm.Dialector) (*gorm.DB, error) {
-	return gorm.Open(sqlite, &gorm.Config{})
+	db, err := gorm.Open(sqlite, &gorm.Config{})
+	if err != nil {
+		return nil, err
+	}
+
+	pragmas := []string{
+		`PRAGMA foreign_keys = ON;`,
+		`PRAGMA journal_mode = WAL;`,
+	}
+
+	for _, pragma := range pragmas {
+		res := db.Exec(pragma)
+		if res.Error != nil {
+			return nil, res.Error
+		}
+	}
+	return db, nil
 }

internal/storage/sqlite/migrations/migrator.go

@@ -0,0 +1,108 @@
+package migrations
+
+import (
+	"database/sql"
+	"fmt"
+	_202409061249_bootstrapDb "github.com/Layr-Labs/sidecar/internal/sqlite/migrations/202409061249_bootstrapDb"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"time"
+)
+
+type ISqliteMigration interface {
+	Up(grm *gorm.DB) error
+	GetName() string
+}
+
+type SqliteMigrator struct {
+	Db     *sql.DB
+	GDb    *gorm.DB
+	Logger *zap.Logger
+}
+
+func NewSqliteMigrator(gDb *gorm.DB, l *zap.Logger) *SqliteMigrator {
+	return &SqliteMigrator{
+		GDb:    gDb,
+		Logger: l,
+	}
+}
+
+func (m *SqliteMigrator) MigrateAll() error {
+	err := m.CreateMigrationTablesIfNotExists()
+	if err != nil {
+		return err
+	}
+
+	migrations := []ISqliteMigration{
+		&_202409061249_bootstrapDb.SqliteMigration{},
+	}
+
+	for _, migration := range migrations {
+		err := m.Migrate(migration)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return nil
+}
+
+func (m *SqliteMigrator) CreateMigrationTablesIfNotExists() error {
+	queries := []string{
+		`CREATE TABLE IF NOT EXISTS migrations (
+			name TEXT PRIMARY KEY,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+			updated_at DATETIME DEFAULT NULL,
+			deleted_at DATETIME DEFAULT NULL
+		)`,
+	}
+
+	for _, query := range queries {
+		res := m.GDb.Exec(query)
+		if res.Error != nil {
+			m.Logger.Sugar().Errorw("Failed to create migration table", zap.Error(res.Error))
+			return res.Error
+		}
+	}
+	return nil
+}
+
+func (m *SqliteMigrator) Migrate(migration ISqliteMigration) error {
+	name := migration.GetName()
+
+	// find migration by name
+	var migrationRecord Migrations
+	result := m.GDb.Find(&migrationRecord, "name = ?", name).Limit(1)
+
+	if result.Error == nil && result.RowsAffected == 0 {
+		m.Logger.Sugar().Infof("Running migration '%s'", name)
+		// run migration
+		err := migration.Up(m.GDb)
+		if err != nil {
+			m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to run migration '%s'", name), zap.Error(err))
+			return err
+		}
+
+		// record migration
+		migrationRecord = Migrations{
+			Name: name,
+		}
+		result = m.GDb.Create(&migrationRecord)
+		if result.Error != nil {
+			m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to record migration '%s'", name), zap.Error(result.Error))
+			return result.Error
+		}
+	} else if result.Error != nil {
+		m.Logger.Sugar().Errorw(fmt.Sprintf("Failed to find migration '%s'", name), zap.Error(result.Error))
+		return result.Error
+	} else if result.RowsAffected > 0 {
+		m.Logger.Sugar().Infof("Migration %s already run", name)
+		return nil
+	}
+	return nil
+}
+
+type Migrations struct {
+	Name      string `gorm:"primaryKey"`
+	CreatedAt time.Time
+	UpdatedAt time.Time
+}

internal/storage/sqlite/sqlite_test.go

@@ -0,0 +1,83 @@
+package sqlite
+
+import (
+	"fmt"
+	"github.com/Layr-Labs/sidecar/internal/config"
+	"github.com/Layr-Labs/sidecar/internal/logger"
+	"github.com/Layr-Labs/sidecar/internal/storage"
+	"github.com/Layr-Labs/sidecar/internal/storage/sqlite/migrations"
+	"github.com/Layr-Labs/sidecar/internal/tests"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"testing"
+	"time"
+)
+
+func setup() (*gorm.DB, *zap.Logger, *config.Config) {
+	cfg := config.NewConfig()
+	l, err := logger.NewLogger(&logger.LoggerConfig{Debug: true})
+	db, err := tests.GetSqliteDatabaseConnection()
+	if err != nil {
+		panic(err)
+	}
+	sqliteMigrator := migrations.NewSqliteMigrator(db, l)
+	if err := sqliteMigrator.MigrateAll(); err != nil {
+		l.Sugar().Fatalw("Failed to migrate", "error", err)
+	}
+	return db, l, cfg
+}
+
+func teardown(db *gorm.DB, l *zap.Logger) {
+	queries := []string{
+		`truncate table blocks cascade`,
+		`truncate table transactions cascade`,
+		`truncate table transaction_logs cascade`,
+		`truncate table transaction_logs cascade`,
+	}
+	for _, query := range queries {
+		res := db.Exec(query)
+		if res.Error != nil {
+			l.Sugar().Errorw("Failed to truncate table", "error", res.Error)
+		}
+	}
+}
+
+func Test_SqliteBlockstore(t *testing.T) {
+	t.Run("Blocks", func(t *testing.T) {
+		db, l, cfg := setup()
+
+		sqliteStore := NewSqliteBlockStore(db, l, cfg)
+
+		t.Run("InsertBlockAtHeight", func(t *testing.T) {
+			block := &storage.Block{
+				Number:    100,
+				Hash:      "some hash",
+				BlockTime: time.Now(),
+			}
+
+			insertedBlock, err := sqliteStore.InsertBlockAtHeight(block.Number, block.Hash, uint64(block.BlockTime.Unix()))
+			if err != nil {
+				t.Errorf("Failed to insert block: %v", err)
+			}
+			assert.NotNil(t, insertedBlock)
+			assert.Equal(t, block.Number, insertedBlock.Number)
+			assert.Equal(t, block.Hash, insertedBlock.Hash)
+		})
+		t.Run("Fail to insert a duplicate block", func(t *testing.T) {
+			block := &storage.Block{
+				Number:    100,
+				Hash:      "some hash",
+				BlockTime: time.Now(),
+			}
+
+			_, err := sqliteStore.InsertBlockAtHeight(block.Number, block.Hash, uint64(block.BlockTime.Unix()))
+			assert.NotNil(t, err)
+			assert.Contains(t, err.Error(), "UNIQUE constraint failed")
+			fmt.Printf("Error: %v\n", err)
+		})
+		t.Run("InsertBlockTransaction", func(t *testing.T) {
+
+		})
+	})
+}

internal/storage/storage.go

@@ -19,11 +19,9 @@ type BlockStore interface {
 
 // Tables
 type Block struct {
-	Id        uint64 `gorm:"type:serial"`
 	Number    uint64
 	Hash      string
 	BlockTime time.Time
-	BlobPath  string
 	CreatedAt time.Time
 	UpdatedAt time.Time
 	DeletedAt time.Time

internal/tests/utils.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"github.com/Layr-Labs/sidecar/internal/config"
 	"github.com/Layr-Labs/sidecar/internal/postgres"
+	sqlite2 "github.com/Layr-Labs/sidecar/internal/sqlite"
 	"gorm.io/gorm"
 )
 
@@ -28,3 +29,13 @@ func GetDatabaseConnection(cfg *config.Config) (*postgres.Postgres, *gorm.DB, er
 	}
 	return db, grm, nil
 }
+
+const sqliteInMemoryPath = "file::memory:?cache=shared"
+
+func GetSqliteDatabaseConnection() (*gorm.DB, error) {
+	db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqliteInMemoryPath))
+	if err != nil {
+		panic(err)
+	}
+	return db, nil
+}

scripts/generateSqliteMigration.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+name=$1
+
+if [[ -z $name ]]; then
+    echo "Usage: $0 <migration_name>"
+    exit 1
+fi
+
+timestamp=$(date +"%Y%m%d%H%M")
+
+migration_name="${timestamp}_${name}"
+
+migrations_dir="./internal/sqlite/migrations/${migration_name}"
+migration_file="${migrations_dir}/up.go"
+
+mkdir -p $migrations_dir || true
+
+# heredoc that creates a migration go file with an Up function
+cat > $migration_file <<EOF
+package _${timestamp}_${name}
+
+import (
+	"database/sql"
+	"gorm.io/gorm"
+)
+
+type SqliteMigration struct {
+}
+
+func (m *SqliteMigration) Up(db *sql.DB, grm *gorm.DB) error {
+	return nil
+}
+
+func (m *SqliteMigration) GetName() string {
+	return "${timestamp}_${name}"
+}
+EOF