instance: add migration System

.github/workflows/build.yml

@@ -39,7 +39,6 @@ jobs:
           build-args: |
             BUILD=${{ steps.vars.outputs.build }}
             GIT_BUILD_HASH=${{ steps.vars.outputs.sha }}
-            CC_GH_COMMIT_SHA=${{ github.event.pull_request.head.sha }}
   build-cli:
     name: Build CLI
     runs-on: ubuntu-latest
@@ -49,5 +48,22 @@ jobs:
         os: [darwin, linux]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
       - run: |
           GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} make bin/gravity-cli
+  build-web:
+    name: Build Web
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: web/package.json
+          cache: "npm"
+          cache-dependency-path: web/package-lock.json
+      - run: make web-install gen-client-ts web-build
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.vscode/settings.json

@@ -3,9 +3,12 @@
         "BOOTSTRAP_ROLES": "dns;dhcp;api;discovery;backup;monitoring;tsdb;tftp",
         "ETCD_ENDPOINT": "localhost:2385",
         "DEBUG": "true",
-        "INSTANCE_IP": "0.0.0.0"
+        "INSTANCE_IP": "0.0.0.0",
+        "CI": "true",
     },
     "go.testFlags": ["-count=1", "-failfast", "-shuffle=on", "-v"],
+    "go.coverOnSingleTest": true,
+    "go.coverOnTestPackage": true,
     "gitlens.autolinks": [
         {
             "alphanumeric": true,

Dockerfile

@@ -1,8 +1,6 @@
 # Stage 1: Build web
 FROM --platform=${BUILDPLATFORM} docker.io/library/node:23.3 AS web-builder
 
-ARG CC_GH_COMMIT_SHA
-
 WORKDIR /work
 
 COPY ./Makefile /work/Makefile

Makefile

@@ -176,5 +176,6 @@ test: internal/resources/macoui internal/resources/blocky internal/resources/tft
 		-covermode=atomic \
 		-count=${TEST_COUNT} \
 		${TEST_FLAGS} \
-		./...  2>&1 | tee test-output
+		$(shell go list ./... | grep -v ./api) \
+			2>&1 | tee test-output
 	go tool cover -html coverage.txt -o coverage.html

go.mod

@@ -6,6 +6,7 @@ toolchain go1.23.0
 
 require (
 	github.com/0xERR0R/blocky v0.9.2-0.20241022123918-76aa6cc84cd1
+	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Netflix/go-env v0.1.2
 	github.com/Ullaakut/nmap/v2 v2.2.2
 	github.com/aeden/traceroute v0.0.0-20210211061815-03f5f7cb7908

pkg/extconfig/config_test.go

@@ -25,9 +25,5 @@ func TestGetIP(t *testing.T) {
 }
 
 func TestVersion(t *testing.T) {
-	assert.Equal(t, Version+"-dev", FullVersion())
-	BuildHash = "foo"
-	assert.Equal(t, Version+"-foo", FullVersion())
-	BuildHash = "foobqerqewrqwer"
-	assert.Equal(t, Version+"-foobqerq", FullVersion())
+	assert.Equal(t, Version+"+test", FullVersion())
 }

pkg/extconfig/log_iml/log_iml.go

@@ -7,13 +7,13 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-type inMemoryLogger struct {
+type InMemoryLogger struct {
 	msgs []zapcore.Entry
 	msgM sync.RWMutex
 	max  int
 }
 
-func (iml *inMemoryLogger) Hook() zap.Option {
+func (iml *InMemoryLogger) Hook() zap.Option {
 	return zap.Hooks(func(e zapcore.Entry) error {
 		iml.msgM.Lock()
 		defer iml.msgM.Unlock()
@@ -25,26 +25,31 @@ func (iml *inMemoryLogger) Hook() zap.Option {
 	})
 }
 
-func (iml *inMemoryLogger) Messages() []zapcore.Entry {
+func (iml *InMemoryLogger) MaxSize() int {
+	return iml.max
+}
+
+func (iml *InMemoryLogger) Flush() {
+	iml.msgM.Lock()
+	iml.msgs = make([]zapcore.Entry, 0)
+	iml.msgM.Unlock()
+}
+
+func (iml *InMemoryLogger) Messages() []zapcore.Entry {
 	iml.msgM.RLock()
 	defer iml.msgM.RUnlock()
 	return iml.msgs
 }
 
-var iml *inMemoryLogger
+var iml *InMemoryLogger
 
 func init() {
-	iml = &inMemoryLogger{
+	iml = &InMemoryLogger{
 		msgs: make([]zapcore.Entry, 0),
 		max:  300,
 	}
 }
 
-type InMemoryLogger interface {
-	Messages() []zapcore.Entry
-	Hook() zap.Option
-}
-
-func Get() InMemoryLogger {
+func Get() *InMemoryLogger {
 	return iml
 }

pkg/extconfig/log_iml/log_iml_test.go

@@ -0,0 +1,29 @@
+package log_iml_test
+
+import (
+	"testing"
+
+	"beryju.io/gravity/pkg/extconfig"
+	"beryju.io/gravity/pkg/extconfig/log_iml"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInMemoryLogger(t *testing.T) {
+	iml := log_iml.Get()
+	iml.Flush()
+	extconfig.Get().Logger().Debug("test")
+	msgs := iml.Messages()
+	assert.Len(t, msgs, 1)
+}
+
+func TestInMemoryLogger_Trunc(t *testing.T) {
+	iml := log_iml.Get()
+	iml.Flush()
+	for i := 0; i <= iml.MaxSize(); i++ {
+		extconfig.Get().Logger().Debug("test")
+	}
+	// Log one more message
+	extconfig.Get().Logger().Debug("test")
+	msgs := iml.Messages()
+	assert.Len(t, msgs, iml.MaxSize())
+}

pkg/extconfig/version.go

@@ -1,6 +1,7 @@
 package extconfig
 
 import (
+	"os"
 	"strings"
 )
 
@@ -11,10 +12,14 @@ var (
 )
 
 func FullVersion() string {
+	if os.Getenv("CI") == "true" {
+		Version = "99.99.99"
+		BuildHash = "test"
+	}
 	version := strings.Builder{}
 	version.WriteString(Version)
 	if BuildHash != "" {
-		version.WriteRune('-')
+		version.WriteRune('+')
 		if len(BuildHash) >= 8 {
 			version.WriteString(BuildHash[:8])
 		} else {

pkg/instance/instance.go

@@ -336,7 +336,16 @@
 	defer srs.Finish()
 	defer i.putInstanceInfo(srs.Context())
 	instanceRoleStarted.WithLabelValues(id).SetToCurrentTime()
-	err := i.roles[id].Role.Start(srs.Context(), rawConfig)
+	// Run migrations
+	client, err := i.roles[id].RoleInstance.Migrator().Run(srs.Context())
+	if err != nil {
+		i.log.Warn("failed to run migrations for role", zap.String("roleId", id))
+		return false
+	}
+	// Overwrite role's KV client with the potentially hooked client for migrations
+	i.roles[id].RoleInstance.kv = client
+	// Start role
+	err = i.roles[id].Role.Start(srs.Context(), rawConfig)
 	if err == roles.ErrRoleNotConfigured {
 		i.log.Info("role not configured", zap.String("roleId", id))
 	} else if err != nil {

pkg/instance/migrate/inline_migration.go

@@ -0,0 +1,43 @@
+package migrate
+
+import (
+	"context"
+
+	"beryju.io/gravity/pkg/storage"
+	"github.com/Masterminds/semver/v3"
+)
+
+func MustParseConstraint(input string) *semver.Constraints {
+	c, err := semver.NewConstraint(input)
+	if err != nil {
+		panic(err)
+	}
+	return c
+}
+
+type InlineMigration struct {
+	MigrationName     string
+	ActivateOnVersion *semver.Constraints
+	HookFunc          func(context.Context) (*storage.Client, error)
+	CleanupFunc       func(context.Context) error
+}
+
+func (im *InlineMigration) Name() string {
+	return im.MigrationName
+}
+
+func (im *InlineMigration) Check(clusterVersion *semver.Version, ctx context.Context) (bool, error) {
+	check := im.ActivateOnVersion.Check(clusterVersion)
+	return check, nil
+}
+
+func (im *InlineMigration) Hook(ctx context.Context) (*storage.Client, error) {
+	return im.HookFunc(ctx)
+}
+
+func (im *InlineMigration) Cleanup(ctx context.Context) error {
+	if im.CleanupFunc != nil {
+		return im.CleanupFunc(ctx)
+	}
+	return nil
+}

pkg/instance/migrate/migrate.go

@@ -0,0 +1,104 @@
+package migrate
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+
+	"beryju.io/gravity/pkg/extconfig"
+	"beryju.io/gravity/pkg/instance/types"
+	"beryju.io/gravity/pkg/roles"
+	"beryju.io/gravity/pkg/storage"
+	"github.com/Masterminds/semver/v3"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+type Migrator struct {
+	ri         roles.Instance
+	log        *zap.Logger
+	migrations []roles.Migration
+}
+
+func New(ri roles.Instance) *Migrator {
+	return &Migrator{
+		ri:         ri,
+		log:        ri.Log().Named("migrator"),
+		migrations: make([]roles.Migration, 0),
+	}
+}
+
+func (mi *Migrator) GetClusterVersion() (*semver.Version, error) {
+	type partialInstanceInfo struct {
+		Version string `json:"version" required:"true"`
+	}
+	instances, err := mi.ri.KV().Get(
+		context.Background(),
+		mi.ri.KV().Key(
+			types.KeyInstance,
+		).Prefix(true).String(),
+		clientv3.WithPrefix(),
+	)
+	if err != nil {
+		return nil, err
+	}
+	// Gather all instances in the cluster and parse their versions
+	version := []*semver.Version{}
+	for _, inst := range instances.Kvs {
+		pi := partialInstanceInfo{}
+		err = json.Unmarshal(inst.Value, &pi)
+		if err != nil {
+			mi.log.Warn("failed to parse instance info", zap.Error(err))
+			continue
+		}
+		v, err := semver.NewVersion(pi.Version)
+		if err != nil {
+			mi.log.Warn("failed to parse instance version", zap.Error(err))
+			continue
+		}
+		version = append(version, v)
+	}
+	sort.Sort(semver.Collection(version))
+	if len(version) < 1 {
+		return semver.MustParse(extconfig.FullVersion()), nil
+	}
+	return version[0], nil
+}
+
+func (mi *Migrator) Run(ctx context.Context) (*storage.Client, error) {
+	cv, err := mi.GetClusterVersion()
+	if err != nil {
+		return nil, err
+	}
+	mi.log.Debug("Checking migrations to activate for cluster version", zap.String("clusterVersion", cv.String()))
+	cli := mi.ri.KV()
+	for _, m := range mi.migrations {
+		mi.log.Debug("Checking if migration needs to be run", zap.String("migration", m.Name()))
+		enabled, err := m.Check(cv, ctx)
+		if err != nil {
+			mi.log.Warn("failed to check if migration should be enabled", zap.String("migration", m.Name()), zap.Error(err))
+			return nil, err
+		}
+		if enabled {
+			_cli, err := m.Hook(ctx)
+			if err != nil {
+				mi.log.Warn("failed to hook for migration", zap.String("migration", m.Name()), zap.Error(err))
+				return nil, err
+			}
+			mi.log.Info("Enabling migration", zap.String("migration", m.Name()))
+			cli = _cli
+		} else {
+			mi.log.Info("Running cleanup for migration", zap.String("migration", m.Name()))
+			err := m.Cleanup(ctx)
+			if err != nil {
+				mi.log.Warn("failed to cleanup migration", zap.String("migration", m.Name()), zap.Error(err))
+				continue
+			}
+		}
+	}
+	return cli, nil
+}
+
+func (mi *Migrator) AddMigration(migration roles.Migration) {
+	mi.migrations = append(mi.migrations, migration)
+}