Security: Encrypting job credentials (#13)

* Split job file into multiple files for better overview

* Added security package with encryptor to encode and decode sensitive data

* Implemented HTTP job encoding

* Added AMQP job url validation

* Updated components set encryptor at boot based on the environment. Added compose environment variables. Services are now initialized properly on boot.

* Added base64 encoding and decoding on top of encryption

* Added AMQP job credential encryption.

* Removing credentials from API responses, for safety reasons. The connection URL in AMQP however, will have its username/password removed.

* Updated test workflow

* Added AMQP job conversion test

.github/workflows/test.yaml

@@ -23,7 +23,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version: "1.22.1"
 
       - name: Install dependencies and run tests
         run: |

cmd/manager/main.go

@@ -11,11 +11,15 @@ import (
 	"github.com/GLCharge/otelzap"
 	"github.com/ardanlabs/conf/v3"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	devxCfg "github.com/xBlaz3kx/DevX/configuration"
 	devxHttp "github.com/xBlaz3kx/DevX/http"
 	"github.com/xBlaz3kx/DevX/observability"
 	api "github.com/xBlaz3kx/distributed-scheduler/internal/api/http"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/database"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/logger"
+	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/security"
+	"github.com/xBlaz3kx/distributed-scheduler/internal/store/postgres"
 	"go.uber.org/zap"
 )
 
@@ -46,11 +50,20 @@ type config struct {
 var rootCmd = &cobra.Command{
 	Use:   "scheduler",
 	Short: "Scheduler manager",
-	Run:   runCmd,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.SetDefault("storage.encryption.key", "ishouldreallybechanged")
+		devxCfg.InitConfig("", "./config", ".")
+
+		postgres.SetEncryptor(security.NewEncryptorFromEnv())
+	},
+	Run: runCmd,
 }
 
 func main() {
-	cobra.OnInitialize(logger.SetupLogging)
+	cobra.OnInitialize(func() {
+		logger.SetupLogging()
+		devxCfg.SetupEnv("manager")
+	})
 	err := rootCmd.Execute()
 	if err != nil {
 		panic(err)

cmd/runner/main.go

@@ -12,11 +12,14 @@ import (
 	"github.com/GLCharge/otelzap"
 	"github.com/ardanlabs/conf/v3"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	devxCfg "github.com/xBlaz3kx/DevX/configuration"
 	devxHttp "github.com/xBlaz3kx/DevX/http"
 	"github.com/xBlaz3kx/DevX/observability"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/executor"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/database"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/logger"
+	"github.com/xBlaz3kx/distributed-scheduler/internal/pkg/security"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/runner"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/service/job"
 	"github.com/xBlaz3kx/distributed-scheduler/internal/store/postgres"
@@ -26,7 +29,7 @@ import (
 var build = "develop"
 
 var serviceInfo = observability.ServiceInfo{
-	Name:    "manager",
+	Name:    "runner",
 	Version: build,
 }
 
@@ -49,11 +52,20 @@ type config struct {
 var rootCmd = &cobra.Command{
 	Use:   "runner",
 	Short: "Scheduler runner",
-	Run:   runCmd,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.SetDefault("storage.encryption.key", "ishouldreallybechanged")
+		devxCfg.InitConfig("", "./config", ".")
+
+		postgres.SetEncryptor(security.NewEncryptorFromEnv())
+	},
+	Run: runCmd,
 }
 
 func main() {
-	cobra.OnInitialize(logger.SetupLogging)
+	cobra.OnInitialize(func() {
+		logger.SetupLogging()
+		devxCfg.SetupEnv("runner")
+	})
 	err := rootCmd.Execute()
 	if err != nil {
 		panic(err)

docker-compose.local-dev.yml

@@ -29,6 +29,7 @@ services:
       - MANAGER_DB_USER=scheduler
       - MANAGER_DB_PASS=scheduler
       - MANAGER_DB_DISABLETLS=true
+      - MANAGER_STORAGE_ENCRYPTION_KEY=ishouldbechanged
     depends_on:
       - postgres
       - migration
@@ -44,6 +45,7 @@ services:
       - RUNNER_DB_USER=scheduler
       - RUNNER_DB_PASS=scheduler
       - RUNNER_DB_DISABLETLS=true
+      - RUNNER_STORAGE_ENCRYPTION_KEY=ishouldbechanged
     depends_on:
       - postgres
       - migration
@@ -57,6 +59,8 @@ services:
       POSTGRES_DB: scheduler
     volumes:
       - pgdata:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
 
   rabbitmq:
     image: rabbitmq:3-management

go.mod

@@ -8,68 +8,106 @@ require (
 	github.com/GLCharge/otelzap v0.0.0-20230904131944-57dc7c9994a9
 	github.com/ardanlabs/darwin/v3 v3.3.1
 	github.com/cenkalti/backoff/v4 v4.3.0
-	github.com/gin-contrib/zap v1.1.3
 	github.com/google/go-cmp v0.6.0
 	github.com/lib/pq v1.10.9
 	github.com/samber/lo v1.46.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.19.0
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
-	github.com/vearne/gin-timeout v0.2.0
+	github.com/swaggo/swag v1.16.3
 	github.com/xBlaz3kx/DevX v0.0.0-20240731212815-b4dca2816bed
 )
 
 require (
+	cloud.google.com/go v0.112.1 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/firestore v1.15.0 // indirect
+	cloud.google.com/go/longrunning v0.5.5 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
+	github.com/agrison/go-commons-lang v0.0.0-20240106075236-2e001e6401ef // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fvbock/endless v0.0.0-20170109170031-447134032cb6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
+	github.com/gin-contrib/zap v1.1.3 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.8 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
 	github.com/grafana/otel-profiling-go v0.5.1 // indirect
 	github.com/grafana/pyroscope-go v1.1.1 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/hashicorp/consul/api v1.28.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.13.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/nats-io/nats.go v1.34.0 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oapi-codegen/runtime v1.0.0 // indirect
 	github.com/redis/go-redis/v9 v9.5.3 // indirect
+	github.com/sagikazarmark/crypt v0.19.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/swaggo/swag v1.16.3 // indirect
 	github.com/tavsec/gin-healthcheck v1.6.2 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.3 // indirect
+	github.com/vearne/gin-timeout v0.2.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.etcd.io/etcd/api/v3 v3.5.12 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.12 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.12 // indirect
 	go.mongodb.org/mongo-driver v1.15.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/contrib/propagators/b3 v1.28.0 // indirect
 	go.opentelemetry.io/contrib/propagators/jaeger v1.23.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
@@ -82,8 +120,12 @@ require (
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/oauth2 v0.20.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/api v0.171.0 // indirect
+	google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/grpc v1.65.0 // indirect