FS-1234; Expand DB Schema to allow storing Config Sets

.github/workflows/lint-test.yml

@@ -21,7 +21,7 @@ jobs:
           go-version: '1.22.1'
 
       - name: Install cockroach binary
-        run: curl https://binaries.cockroachdb.com/cockroach-v23.1.11.linux-amd64.tgz | tar -xz && sudo cp -i cockroach-v23.1.11.linux-amd64/cockroach /usr/local/bin/
+        run: curl https://binaries.cockroachdb.com/cockroach-v21.1.21.linux-amd64.tgz | tar -xz && sudo cp -i cockroach-v21.1.21.linux-amd64/cockroach /usr/local/bin/
 
       - name: Start test database
         run: cockroach start-single-node --insecure --background

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM golang:1.22-alpine3.19
 
 # Copy the binary that goreleaser built
 COPY fleetdb /fleetdb

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.19 as builder
+FROM golang:1.22 as builder
 
 # Create and change to the app directory.
 WORKDIR /app

Makefile

@@ -14,13 +14,18 @@ test: | unit-test integration-test
 ## run integration tests
 integration-test: test-database
 	@echo Running integration tests...
-	@FLEETDB_CRDB_URI="${TEST_DB}" go test -cover -tags testtools,integration -p 1 -timeout 1m ./... 
+	@FLEETDB_CRDB_URI="${TEST_DB}" go test -cover -tags testtools,integration -p 1 -timeout 1m ./... | \
+	grep -v "could not be registered in Prometheus\" error=\"duplicate metrics collector registration attempted\"" # TODO; Figure out why this message spams when tests fail
 
-## run lint and unit tests
-unit-test: | lint
+## run unit tests
+unit-test: | test-database
 	@echo Running unit tests...
 	@FLEETDB_CRDB_URI="${TEST_DB}" go test -cover -short -tags testtools ./...
 
+## run single integration test. Example: make single-test test=TestIntegrationServerListComponents
+single-test:
+	@FLEETDB_CRDB_URI="${TEST_DB}" go test -timeout 30s -tags testtools -run ^${test}$$ github.com/metal-toolbox/fleetdb/pkg/api/v1 -v
+
 ## check test coverage
 coverage: | test-database
 	@echo Generating coverage report...
@@ -29,19 +34,21 @@ coverage: | test-database
 	@go tool cover -html=coverage.out
 
 ## lint
-lint: golint
-
-golint: | vendor
+lint: | vendor
 	@echo Linting Go files...
 	@golangci-lint run
 
 ## clean docker files
-clean: docker-clean
+clean: docker-clean test-clean
 	@echo Cleaning...
 	@rm -rf ./dist/
 	@rm -rf coverage.out
+
+## clean test env
+test-clean:
 	@go clean -testcache
 
+## download/tidy go modules
 vendor:
 	@go mod download
 	@go mod tidy
@@ -71,6 +78,22 @@ test-database: | vendor
 	@FLEETDB_CRDB_URI="${TEST_DB}" go run main.go migrate up
 	@cockroach sql --insecure -e "use fleetdb_test; ALTER TABLE attributes DROP CONSTRAINT check_server_id_server_component_id; ALTER TABLE versioned_attributes DROP CONSTRAINT check_server_id_server_component_id;"
 
+## purge dev environment, build new image, and run tests
+fresh-test: clean
+	@make push-image-devel
+	@make docker-up
+	@make test
+
+## boil sql
+boil:
+	make docker-up
+	make test-database
+	sqlboiler crdb --add-soft-deletes
+
+## log into database
+psql:
+	@psql -d "${TEST_DB}"
+
 ## Build linux bin
 build-linux:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${PROJECT_NAME}

README.md

@@ -36,7 +36,7 @@ The `quickstart.yml` compose file will run server service from released images a
 docker-compose -f quickstart.yml -f quickstart-dev.yml up --build
 ```
 
-NOTE: `--build` is required to get docker-compose to rebuild the container if you have changes. You make also include the `quickstart-tracing.yml` file if you wish to have tracing support.
+NOTE: `--build` is required to get docker-compose to rebuild the container if you have changes. You may also include the `quickstart-tracing.yml` file if you wish to have tracing support.
 
 
 ### Adding/Changing database schema

db/migrations/00002_config_sets.sql

@@ -0,0 +1,43 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE public.bios_config_sets (
+    id UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
+    name STRING UNIQUE NOT NULL,
+    version STRING NOT NULL,
+    created_at TIMESTAMPTZ NULL,
+    updated_at TIMESTAMPTZ NULL
+);
+
+CREATE TABLE public.bios_config_components (
+    id UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
+    fk_bios_config_set_id UUID NOT NULL REFERENCES public.bios_config_sets(id) ON DELETE CASCADE,
+    name STRING NOT NULL,
+    vendor STRING NOT NULL,
+    model STRING NOT NULL,
+    created_at TIMESTAMPTZ NULL,
+    updated_at TIMESTAMPTZ NULL,
+    UNIQUE (fk_bios_config_set_id, name)
+);
+
+CREATE TABLE public.bios_config_settings (
+    id UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
+    fk_bios_config_component_id UUID NOT NULL REFERENCES public.bios_config_components(id) ON DELETE CASCADE,
+    settings_key STRING NOT NULL,
+    settings_value STRING NOT NULL,
+    raw JSONB NULL,
+    created_at TIMESTAMPTZ NULL,
+    updated_at TIMESTAMPTZ NULL,
+    UNIQUE (fk_bios_config_component_id, settings_key)
+);
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+
+DROP TABLE public.bios_config_settings;
+DROP TABLE public.bios_config_components;
+DROP TABLE public.bios_config_sets;
+
+-- +goose StatementEnd

go.mod

@@ -35,6 +35,7 @@ require (
 
 require (
 	github.com/bmc-toolbox/common v0.0.0-20231204194243-7bcbccab7116
+	github.com/hetiansu5/urlquery v1.2.7
 	github.com/metal-toolbox/rivets v1.0.3
 	github.com/volatiletech/sqlboiler v3.7.1+incompatible
 	go.hollow.sh/toolbox v0.6.3

go.sum

@@ -368,6 +368,8 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
 github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
+github.com/hetiansu5/urlquery v1.2.7 h1:jn0h+9pIRqUziSPnRdK/gJK8S5TCnk+HZZx5fRHf8K0=
+github.com/hetiansu5/urlquery v1.2.7/go.mod h1:wFpZdTHRdwt7mk0EM/DdZEWtEN4xf8HJoH/BLXm/PG0=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

internal/dbtools/fixtures.go

@@ -89,6 +89,10 @@ var (
 
 	// Inventory fixtures
 	FixtureInventoryServer *models.Server
+
+	FixtureBiosConfigSet        *models.BiosConfigSet
+	FixtureBiosConfigComponents []*models.BiosConfigComponent
+	FixtureBiosConfigSettings   [][]*models.BiosConfigSetting
 )
 
 func addFixtures(t *testing.T) error {
@@ -151,6 +155,10 @@ func addFixtures(t *testing.T) error {
 		return err
 	}
 
+	if err := setupConfigSet(ctx, testDB); err != nil {
+		return err
+	}
+
 	// excluding Chuckles here since that server is deleted
 	FixtureServers = models.ServerSlice{FixtureNemo, FixtureDory, FixtureMarlin}
 	FixtureDeletedServers = models.ServerSlice{FixtureChuckles}
@@ -649,3 +657,81 @@ func setupInventoryFixture(ctx context.Context, db *sqlx.DB) error {
 
 	return nil
 }
+
+func setupConfigSet(ctx context.Context, db *sqlx.DB) error {
+	settings := [][]*models.BiosConfigSetting{
+		{
+			{
+				SettingsKey:   "BootOrder",
+				SettingsValue: "dev2,dev3,dev4",
+			},
+			{
+				SettingsKey:   "Mode",
+				SettingsValue: "UEFI",
+			},
+		},
+		{
+			{
+				SettingsKey:   "PXEEnable",
+				SettingsValue: "true",
+				Raw:           null.NewJSON([]byte(`{}`), true),
+			},
+			{
+				SettingsKey:   "SRIOVEnable",
+				SettingsValue: "false",
+			},
+			{
+				SettingsKey:   "position",
+				SettingsValue: "1",
+				Raw:           null.NewJSON([]byte(`{ "lanes": 8 }`), true),
+			},
+		},
+	}
+
+	components := []*models.BiosConfigComponent{
+		{
+			Name:   "Fixture Test SM Motherboard",
+			Vendor: "SUPERMICRO",
+			Model:  "ATX",
+		},
+		{
+			Name:   "Fixture Test Intel Network Adapter",
+			Vendor: "Intel",
+			Model:  "PCIE",
+		},
+	}
+
+	configSet := models.BiosConfigSet{
+		Name:    "Fixture Test Config Set",
+		Version: "version",
+	}
+
+	err := configSet.Insert(ctx, db, boil.Infer())
+	if err != nil {
+		return err
+	}
+
+	for c := range components {
+		components[c].FKBiosConfigSetID = configSet.ID
+
+		err = configSet.AddFKBiosConfigSetBiosConfigComponents(ctx, db, true, components[c])
+		if err != nil {
+			return err
+		}
+
+		for s := range settings[c] {
+			settings[c][s].FKBiosConfigComponentID = components[c].ID
+		}
+
+		err = components[c].AddFKBiosConfigComponentBiosConfigSettings(ctx, db, true, settings[c]...)
+		if err != nil {
+			return err
+		}
+	}
+
+	FixtureBiosConfigSet = &configSet
+	FixtureBiosConfigComponents = components
+	FixtureBiosConfigSettings = settings
+
+	return nil
+}

internal/dbtools/testtools.go

@@ -111,6 +111,7 @@ func cleanDB(t *testing.T) {
 	if _, err := models.Servers(qm.WithDeleted()).DeleteAll(ctx, boil.GetContextDB(), true); err != nil {
 		t.Error(errors.Wrap(err, "table: model.Servers"))
 	}
+
 	deleteFixture(ctx, t, models.AttributesFirmwareSets())
 	deleteFixture(ctx, t, models.ComponentFirmwareSets())
 	deleteFixture(ctx, t, models.ComponentFirmwareSetMaps())
@@ -122,6 +123,10 @@ func cleanDB(t *testing.T) {
 	deleteFixture(ctx, t, models.BMCMacAddresses())
 	deleteFixture(ctx, t, models.BomInfos())
 
+	deleteFixture(ctx, t, models.BiosConfigSettings())
+	deleteFixture(ctx, t, models.BiosConfigComponents())
+	deleteFixture(ctx, t, models.BiosConfigSets())
+
 	testDB.Exec("SET sql_safe_updates = true;")
 }