From b7509ee5c5fd498ba460b36bbc1a91f612ce16ad Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 15 Apr 2024 18:58:32 -0600 Subject: [PATCH 1/4] save: wip to add ssl, not working yet Signed-off-by: vsoch --- Makefile | 27 ++++-- cmd/rainbow/config/config.go | 2 + cmd/rainbow/rainbow.go | 39 +++++++-- cmd/rainbow/receive/receive.go | 9 +- cmd/rainbow/register/register.go | 9 +- cmd/rainbow/register/subsystem.go | 8 +- cmd/rainbow/submit/submit.go | 9 +- cmd/rainbow/update/state.go | 7 +- cmd/server/server.go | 33 ++++++- docs/auth.md | 80 +++++++++++++++++ docs/sidebar.md | 1 + hack/client-ext.conf | 1 + hack/generate-certs.sh | 48 +++++++++++ hack/server-ext.conf | 1 + pkg/certs/certs.go | 139 ++++++++++++++++++++++++++++++ pkg/client/client.go | 29 +++++-- pkg/server/server.go | 15 +++- 17 files changed, 403 insertions(+), 54 deletions(-) create mode 100644 docs/auth.md create mode 100644 hack/client-ext.conf create mode 100755 hack/generate-certs.sh create mode 100644 hack/server-ext.conf create mode 100644 pkg/certs/certs.go diff --git a/Makefile b/Makefile index 037d857..d1343c5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ HERE ?= $(shell pwd) LOCALBIN ?= $(shell pwd)/bin +CERTBIN ?= $(LOCALBIN)/certs VERSION :=$(shell cat .version) YAML_FILES :=$(shell find . ! -path "./vendor/*" -type f -regex ".*y*ml" -print) REGISTRY ?= ghcr.io/converged-computing @@ -11,39 +12,47 @@ all: help $(LOCALBIN): mkdir -p $(LOCALBIN) +.PHONY: $(CERTBIN) +$(CERTBIN): + mkdir -p $(CERTBIN) + .PHONY: protoc protoc: $(LOCALBIN) GOBIN=$(LOCALBIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 GOBIN=$(LOCALBIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 -.PHONY: build +.PHONY: build ## Build client and server build: build-cli build-rainbow .PHONY: build-cli -build-cli: $(LOCALBIN) +build-cli: $(LOCALBIN) ## Build rainbow Go client GO111MODULE="on" go build -o $(LOCALBIN)/rainbow cmd/rainbow/rainbow.go .PHONY: build-rainbow -build-rainbow: $(LOCALBIN) +build-rainbow: $(LOCALBIN) ## Build rainbow scheduler (server) GO111MODULE="on" go build -o $(LOCALBIN)/rainbow-scheduler cmd/server/server.go -.PHONY: docker +.PHONY: docker ## Make all docker images docker: docker-flux docker-ubuntu .PHONY: docker-flux -docker-flux: +docker-flux: ## Make docker ubuntu + flux image docker build --build-arg base=fluxrm/flux-sched:jammy -t $(REGISTRY)/rainbow-flux:latest . .PHONY: docker-ubuntu -docker-ubuntu: +docker-ubuntu: ## Make docker ubuntu images docker build -t $(REGISTRY)/rainbow-scheduler:latest . .PHONY: docker-arm docker-arm: docker buildx build --build-arg arch=arm64 --platform linux/arm64 --tag $(REGISTRY)/rainbow-scheduler:arm --load . +.PHONY: certs +certs: $(CERTBIN) ## Make self-signed certificates + $(HERE)/hack/generate-certs.sh $(CERTBIN) + .PHONY: proto -proto: protoc ## Generates the API code and documentation +proto: protoc ## Make protobuf files mkdir -p pkg/api/v1 PATH=$(LOCALBIN):${PATH} protoc --proto_path=api/v1 --go_out=pkg/api/v1 --go_opt=paths=source_relative --go-grpc_out=pkg/api/v1 --go-grpc_opt=paths=source_relative rainbow.proto PATH=$(LOCALBIN):${PATH} protoc --proto_path=plugins/backends/memory/service --go_out=plugins/backends/memory/service --go_opt=paths=source_relative --go-grpc_out=plugins/backends/memory/service --go-grpc_opt=paths=source_relative memory.proto @@ -83,6 +92,10 @@ test: tidy ## Runs unit tests server: ## Runs uncompiled version of the server go run cmd/server/server.go --global-token rainbow +.PHONY: server-tls +server-tls: ## Runs uncompiled version of the server with self-signed certs + go run cmd/server/server.go --global-token rainbow -cert $(CERTBIN)/server-cert.pem -ca-cert $(CERTBIN)/ca-cert.pem --key $(CERTBIN)/server-key.pem + .PHONY: server-verbose server-verbose: ## Runs uncompiled version of the server go run cmd/server/server.go --loglevel 6 --global-token rainbow diff --git a/cmd/rainbow/config/config.go b/cmd/rainbow/config/config.go index 93074f3..f92770b 100644 --- a/cmd/rainbow/config/config.go +++ b/cmd/rainbow/config/config.go @@ -4,6 +4,7 @@ import ( "log" "os" + "github.com/converged-computing/rainbow/pkg/certs" "github.com/converged-computing/rainbow/pkg/config" "gopkg.in/yaml.v3" ) @@ -16,6 +17,7 @@ var ( func RunInit( path string, clusterName, selectAlgo, matchAlgo string, + cert *certs.Certificate, ) error { if path == "" { diff --git a/cmd/rainbow/rainbow.go b/cmd/rainbow/rainbow.go index a4b23aa..f17e8b4 100644 --- a/cmd/rainbow/rainbow.go +++ b/cmd/rainbow/rainbow.go @@ -11,6 +11,8 @@ import ( "github.com/converged-computing/rainbow/cmd/rainbow/register" "github.com/converged-computing/rainbow/cmd/rainbow/submit" "github.com/converged-computing/rainbow/cmd/rainbow/update" + "github.com/converged-computing/rainbow/pkg/certs" + "github.com/converged-computing/rainbow/pkg/client" "github.com/converged-computing/rainbow/pkg/types" // Register database backends and selection algorithms @@ -51,6 +53,11 @@ func main() { configInitCmd := configCmd.NewCommand("init", "Create a new configuration file") cfg := parser.String("", "config-path", &argparse.Options{Help: "Configuration file for cluster credentials"}) + // Credentials for client tls + caCertFile := parser.String("", "ca-cert", &argparse.Options{Help: "Client CA cert file"}) + certFile := parser.String("", "cert", &argparse.Options{Help: "Client cert file"}) + keyFile := parser.String("", "key", &argparse.Options{Help: "Client key file"}) + // Shared values host := parser.String("", "host", &argparse.Options{Default: "localhost:50051", Help: "Scheduler server address (host:port)"}) clusterName := parser.String("", "cluster-name", &argparse.Options{Help: "Name of cluster to register"}) @@ -93,15 +100,33 @@ func main() { return } + // TODO: refactor so configuration is generated at this level + // and we don't need to pass all the custom parameters to each client function + + // Generate certificate manager + cert, err := certs.NewClientCertificate(*caCertFile, *certFile, *keyFile) + if err != nil { + log.Fatalf("error creating certificate manager: %v", err) + } + + // Config is the only command that doesn't require the client if configCmd.Happened() && configInitCmd.Happened() { - err := config.RunInit(*cfg, *clusterName, *selectAlgo, *matchAlgo) + err := config.RunInit(*cfg, *clusterName, *selectAlgo, *matchAlgo, cert) if err != nil { log.Fatalf("Issue with config: %s\n", err) } + return + } + + // Create the client to be used across calls + client, err := client.NewClient(*host, cert) + if err != nil { + log.Fatalf("Issue creating client: %s\n", err) + } - } else if stateCmd.Happened() { + if stateCmd.Happened() { err := update.UpdateState( - *host, + client, *clusterName, *stateFile, *cfg, @@ -114,7 +139,7 @@ func main() { if subsysCmd.Happened() { err := register.RegisterSubsystem( - *host, + client, *clusterName, *clusterNodes, *subsystem, @@ -125,7 +150,7 @@ func main() { } } else if registerClusterCmd.Happened() { err := register.Run( - *host, + client, *clusterName, *clusterNodes, *secret, @@ -145,7 +170,7 @@ func main() { } else if receiveCmd.Happened() { err := receive.Run( - *host, + client, *clusterName, *clusterSecret, *maxJobs, @@ -156,7 +181,7 @@ func main() { } } else if submitCmd.Happened() { err := submit.Run( - *host, + client, *jobName, *command, *nodes, diff --git a/cmd/rainbow/receive/receive.go b/cmd/rainbow/receive/receive.go index 11068f8..042161c 100644 --- a/cmd/rainbow/receive/receive.go +++ b/cmd/rainbow/receive/receive.go @@ -11,15 +11,12 @@ import ( // Run will check a manifest list of artifacts against a host machine // For now, the host machine parameters will be provided as flags func Run( - host, cluster, secret string, + c client.Client, + cluster, secret string, maxJobs int, cfgFile string, -) error { - c, err := client.NewClient(host) - if err != nil { - return nil - } +) error { // Note that 0 or below indicates "show all jobs" if maxJobs >= 1 { diff --git a/cmd/rainbow/register/register.go b/cmd/rainbow/register/register.go index 68deb7d..126cb0f 100644 --- a/cmd/rainbow/register/register.go +++ b/cmd/rainbow/register/register.go @@ -12,7 +12,7 @@ import ( // Run will register the cluster with rainbow func Run( - host, + c client.Client, clusterName, clusterNodes, secret string, @@ -22,13 +22,8 @@ func Run( subsystem, selectionAlgorithm string, matchAlgorithm string, -) error { - - c, err := client.NewClient(host) - if err != nil { - return err - } +) error { if clusterName == "" { return fmt.Errorf("s --cluster-name is required") } diff --git a/cmd/rainbow/register/subsystem.go b/cmd/rainbow/register/subsystem.go index 608cd66..07612be 100644 --- a/cmd/rainbow/register/subsystem.go +++ b/cmd/rainbow/register/subsystem.go @@ -11,17 +11,13 @@ import ( // RegisterSubsystem registers a subsystem func RegisterSubsystem( - host, + c client.Client, clusterName, subsystemNodes, subsystem, cfgFile string, -) error { - c, err := client.NewClient(host) - if err != nil { - return err - } +) error { // A config file is required here if cfgFile == "" { diff --git a/cmd/rainbow/submit/submit.go b/cmd/rainbow/submit/submit.go index 0edb3bd..a19bc7e 100644 --- a/cmd/rainbow/submit/submit.go +++ b/cmd/rainbow/submit/submit.go @@ -14,18 +14,15 @@ import ( // Run will check a manifest list of artifacts against a host machine // For now, the host machine parameters will be provided as flags func Run( - host, jobName, command string, + c client.Client, + jobName, command string, nodes, tasks int, token, jobspec, clusterName, database, cfgFile string, selectAlgo, matchAlgo string, ) error { - c, err := client.NewClient(host) - if err != nil { - return nil - } - + var err error jspec := &js.Jobspec{} if jobspec == "" { jspec, err = jscli.JobspecFromCommand(command, jobName, int32(nodes), int32(tasks)) diff --git a/cmd/rainbow/update/state.go b/cmd/rainbow/update/state.go index db357db..cd9bb02 100644 --- a/cmd/rainbow/update/state.go +++ b/cmd/rainbow/update/state.go @@ -11,17 +11,12 @@ import ( // UpdateState updates state for a cluster func UpdateState( - host, + c client.Client, clusterName, stateFile, cfgFile string, ) error { - c, err := client.NewClient(host) - if err != nil { - return err - } - // A config file is required here if cfgFile == "" { return fmt.Errorf("an existing configuration file is required to update an existing cluster") diff --git a/cmd/server/server.go b/cmd/server/server.go index c914d47..3d368c0 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -5,6 +5,7 @@ import ( "flag" "log" + "github.com/converged-computing/rainbow/pkg/certs" "github.com/converged-computing/rainbow/pkg/config" rlog "github.com/converged-computing/rainbow/pkg/logger" "github.com/converged-computing/rainbow/pkg/server" @@ -26,6 +27,9 @@ var ( loggingLevel = 3 name = "rainbow" sqliteFile = "rainbow.db" + caCertFile = "" + certFile = "" + keyFile = "" configFile = "" matchAlgo = "match" selectAlgo = "random" @@ -44,9 +48,13 @@ func main() { flag.StringVar(&database, "graph-database", database, "graph database backend (defaults to memory)") flag.StringVar(&selectAlgo, "select-algorithm", selectAlgo, "selection algorithm for final cluster selection (defaults to random)") flag.StringVar(&matchAlgo, "match-algorithm", matchAlgo, "match algorithm for graph (defaults to random)") + flag.StringVar(&caCertFile, "ca-cert", caCertFile, "Server certificate file for TLS (e.g., ca-cert.pem)") + flag.StringVar(&certFile, "cert", certFile, "Server certificate file for TLS (e.g., server-cert.pem)") + flag.StringVar(&keyFile, "key", keyFile, "Server key file for TLS (e.g., server-key.pem)") flag.StringVar(&configFile, "config", configFile, "rainbow config file") flag.IntVar(&loggingLevel, "loglevel", loggingLevel, "rainbow logging level (0 to 5)") flag.BoolVar(&cleanup, "cleanup", cleanup, "cleanup previous sqlite database (default: false)") + flag.Parse() // If the logging level isn't the default, set it @@ -55,14 +63,35 @@ func main() { } // Load (or generate a default) config file here, if provided - cfg, err := config.NewRainbowClientConfig(configFile, name, secret, database, selectAlgo, matchAlgo) + cfg, err := config.NewRainbowClientConfig( + configFile, + name, + secret, + database, + selectAlgo, + matchAlgo, + ) if err != nil { log.Fatalf("error while creating server: %v", err) } + // Generate certificate manager + cert, err := certs.NewServerCertificate(caCertFile, certFile, keyFile) + if err != nil { + log.Fatalf("error creating certificate manager: %v", err) + } + // create server log.Print("creating 🌈ī¸ server...") - s, err := server.NewServer(cfg, types.Version, sqliteFile, cleanup, globalToken, host) + s, err := server.NewServer( + cfg, + types.Version, + sqliteFile, + cleanup, + globalToken, + host, + cert, + ) if err != nil { log.Fatalf("error while creating server: %v", err) } diff --git a/docs/auth.md b/docs/auth.md new file mode 100644 index 0000000..ae34d43 --- /dev/null +++ b/docs/auth.md @@ -0,0 +1,80 @@ +# Authentication + +Rainbow currently supports two modes of running: + +- No authentication (default) +- With certificates (with `--ssl` flag) + + +Without authentication, just use rainbow as is. + +## With Authentication + +### Self-signed Certificates + +Ideally, you have generated signed certificates for your server. For self signed certificates, use: + +```bash +make certs +``` + +And they will be generate for you in the local bin, both for the server and a client: + +```bash +bin/certs/ +├── ca-cert.pem +├── ca-cert.srl +├── ca-key.pem +├── client-cert.pem +├── client-key.pem +├── client-req.pem +├── server-cert.pem +├── server-key.pem +└── server-req.pem +``` + +### Server with TLS + +Then you can run the server using these paths as the defaults: + +```console +$ make server-tls +``` +```console +go run cmd/server/server.go --global-token rainbow -cert /home/vanessa/Desktop/Code/rainbow/bin/certs/server-cert.pem -ca-cert /home/vanessa/Desktop/Code/rainbow/bin/certs/ca-cert.pem --key /home/vanessa/Desktop/Code/rainbow/bin/certs/server-key.pem +2024/04/15 18:17:46 creating 🌈ī¸ server... +2024/04/15 18:17:46 🧩ī¸ selection algorithm: random +2024/04/15 18:17:46 🧩ī¸ graph database: memory +2024/04/15 18:17:46 ✨ī¸ creating rainbow.db... +2024/04/15 18:17:46 rainbow.db file created +2024/04/15 18:17:46 🏓ī¸ creating tables... +2024/04/15 18:17:46 🏓ī¸ tables created +2024/04/15 18:17:46 ⚠ī¸ WARNING: global-token is set, use with caution. +2024/04/15 18:17:46 starting scheduler server: rainbow v0.1.1-draft +2024/04/15 18:17:46 🔐ī¸ adding tls credentials +2024/04/15 18:17:46 🧠ī¸ Registering memory graph database... +2024/04/15 18:17:46 server listening: [::]:50051 +``` + +Note the command above - you can customize this to use your own paths. Also note that when the server starts with credentials, you will see `🔐ī¸ adding tls credentials`. + +### Client with TLS + +The client takes a similar path, but instead uses the client-*.pem files referenced above. +With the server running, try doing a register without credentials. + +```bash +make register +``` + +WIP not working: https://medium.com/@mertkimyonsen/securing-grpc-connection-with-ssl-tls-certificate-using-go-db3852fe89dd + +It will hang. Likely we can add a timeout to prevent that, but it's not needed for now. Let's add the client credentials and see if we get a different response. + +```bash +go run cmd/rainbow/rainbow.go register cluster --cluster-name keebler --nodes-json ./docs/examples/scheduler/cluster-nodes.json \ + --config-path ./docs/examples/scheduler/rainbow-config.yaml --save \ + --cert /home/vanessa/Desktop/Code/rainbow/bin/certs/client-cert.pem --ca-cert /home/vanessa/Desktop/Code/rainbow/bin/certs/ca-cert.pem --key /home/vanessa/Desktop/Code/rainbow/bin/certs/client-key.pem +``` + +[home](/README.md#rainbow-scheduler) diff --git a/docs/sidebar.md b/docs/sidebar.md index 5a17d2e..f492143 100644 --- a/docs/sidebar.md +++ b/docs/sidebar.md @@ -4,5 +4,6 @@ - [Backends](backends.md) - [Advanced](advanced.md) - [Developer](developer.md) + - [Certificates](auth.md) - [Design](design.md) - [Support](support.md) diff --git a/hack/client-ext.conf b/hack/client-ext.conf new file mode 100644 index 0000000..29ff676 --- /dev/null +++ b/hack/client-ext.conf @@ -0,0 +1 @@ +subjectAltName=DNS:*.someclient.com,IP:0.0.0.0 \ No newline at end of file diff --git a/hack/generate-certs.sh b/hack/generate-certs.sh new file mode 100755 index 0000000..75c2a46 --- /dev/null +++ b/hack/generate-certs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +here=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +root=${1} + +if [ "${root}" == "" ]; then + root=$here +fi + +cd ${root} +echo "Building in ${root}" +ls +sleep 3 + +# delete pem file +rm *.pem + +# Create CA private key and self-signed certificate +# adding -nodes to not encrypt the private key +openssl req -x509 -newkey rsa:4096 -nodes -days 365 -keyout ca-key.pem -out ca-cert.pem -subj "/C=US/ST=California/CN=localhost" + +echo "CA's self-signed certificate" +openssl x509 -in ca-cert.pem -noout -text + +# Create Web Server private key and CSR +# adding -nodes to not encrypt the private key +openssl req -newkey rsa:4096 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=US/ST=California/CN=localhost" + +# Sign the Web Server Certificate Request (CSR) +openssl x509 -req -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extfile $here/server-ext.conf + +echo "Server's signed certificate" +openssl x509 -in server-cert.pem -noout -text + +# Verify certificate +echo "Verifying certificate" +openssl verify -CAfile ca-cert.pem server-cert.pem + +# Generate client's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -keyout client-key.pem -out client-req.pem -subj "/C=US/ST=California/CN=localhost" + +# Sign the Client Certificate Request (CSR) +openssl x509 -req -in client-req.pem -days 60 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extfile $here/client-ext.conf + +echo "Client's signed certificate" +openssl x509 -in client-cert.pem -noout -text + +ls . \ No newline at end of file diff --git a/hack/server-ext.conf b/hack/server-ext.conf new file mode 100644 index 0000000..9f3add8 --- /dev/null +++ b/hack/server-ext.conf @@ -0,0 +1 @@ +subjectAltName=DNS:*localhost,IP:0.0.0.0 \ No newline at end of file diff --git a/pkg/certs/certs.go b/pkg/certs/certs.go new file mode 100644 index 0000000..b93963a --- /dev/null +++ b/pkg/certs/certs.go @@ -0,0 +1,139 @@ +package certs + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "google.golang.org/grpc/credentials" +) + +// Certificate is a manager for a certificate. +// Is it intended to be used as a client OR server but not both +// at the same time. +type Certificate struct { + caCertFile string + certFile string + keyFile string + + certPool *x509.CertPool + serverCert *tls.Certificate + clientCert *tls.Certificate +} + +// IsEmpty determines if the certificate is empty +func (c *Certificate) IsEmpty() bool { + return c.caCertFile == "" && c.certFile == "" && c.keyFile == "" +} + +// Validate all fields are defined +func (c *Certificate) Validate() error { + if c.IsEmpty() { + return nil + } + if c.caCertFile == "" { + return fmt.Errorf("the caCertFile is missing") + } + if c.certFile == "" { + return fmt.Errorf("the certFile is missing") + } + if c.keyFile == "" { + return fmt.Errorf("the keyFile is missing") + } + return nil +} + +func (c *Certificate) CreatePool() error { + + // read ca's cert, verify to client's certificate + caPem, err := os.ReadFile(c.caCertFile) + if err != nil { + return err + } + + // create cert pool and append ca's cert + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caPem) { + return err + } + c.certPool = certPool + return nil +} + +// GetCredentials for the server +func (c *Certificate) GetServerCredentials() credentials.TransportCredentials { + + // configuration of the certificate what we want to + conf := &tls.Config{ + Certificates: []tls.Certificate{*c.serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: c.certPool, + } + return credentials.NewTLS(conf) +} + +// GenerateServerCert generates the server certificate +func (c *Certificate) GenerateServerCert() error { + serverCert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) + if err != nil { + return err + } + c.serverCert = &serverCert + return nil +} + +// NewCertificate generates a client agnostic certificate manager +func NewCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) { + // Cut out early if we aren't using TLS + cert := Certificate{caCertFile: caCertFile, keyFile: keyFile, certFile: certFile} + if cert.IsEmpty() { + return &cert, nil + } + err := cert.Validate() + if err != nil { + return nil, fmt.Errorf("the certificate did not validate: %s", err) + } + + // If we get here, create the server certificate and pool + err = cert.CreatePool() + return &cert, err +} + +func NewClientCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) { + cert, err := NewCertificate(caCertFile, certFile, keyFile) + if err != nil || cert.IsEmpty() { + return cert, err + } + err = cert.GenerateClientCert() + return cert, err +} + +func (c *Certificate) GenerateClientCert() error { + //read client cert + clientCert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) + if err != nil { + return err + } + c.clientCert = &clientCert + return nil +} + +// NewCertificate generates a new certificate holder +func NewServerCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) { + cert, err := NewCertificate(caCertFile, certFile, keyFile) + if err != nil || cert.IsEmpty() { + return cert, err + } + err = cert.GenerateServerCert() + return cert, err +} + +// GenerateClientCert generates the client tls certificate +func (c *Certificate) GetClientCredentials() (credentials.TransportCredentials, error) { + config := &tls.Config{ + Certificates: []tls.Certificate{*c.clientCert}, + RootCAs: c.certPool, + } + return credentials.NewTLS(config), nil +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 894545b..b0890ed 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -8,10 +8,12 @@ import ( js "github.com/compspec/jobspec-go/pkg/nextgen/v1" pb "github.com/converged-computing/rainbow/pkg/api/v1" + "github.com/converged-computing/rainbow/pkg/certs" "github.com/converged-computing/rainbow/pkg/config" "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) @@ -42,7 +44,7 @@ type Client interface { } // NewClient creates a new RainbowClient -func NewClient(host string) (Client, error) { +func NewClient(host string, cert *certs.Certificate) (Client, error) { if host == "" { return nil, errors.New("host is required") } @@ -51,12 +53,29 @@ func NewClient(host string) (Client, error) { c := &RainbowClient{host: host} // The context allows us to control the timeout - ctx, cancel := context.WithTimeout(context.TODO(), defaultTimeout) - defer cancel() + // ctx, cancel := context.WithTimeout(context.TODO(), defaultTimeout) + // defer cancel() // Set up a connection to the server. - creds := grpc.WithTransportCredentials(insecure.NewCredentials()) - conn, err := grpc.DialContext(ctx, c.GetHost(), creds, grpc.WithBlock()) + // creds := grpc.WithTransportCredentials(insecure.NewCredentials()) + // conn, err := grpc.DialContext(ctx, c.GetHost(), creds, grpc.WithBlock()) + // Are we using tls? + var transportCreds credentials.TransportCredentials + var err error + if !cert.IsEmpty() { + log.Printf("🔐ī¸ adding tls credentials") + transportCreds, err = cert.GetClientCredentials() + if err != nil { + return nil, err + } + } else { + transportCreds = insecure.NewCredentials() + } + + // Set up a connection to the server. + creds := grpc.WithTransportCredentials(transportCreds) + conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) + if err != nil { return nil, errors.Wrapf(err, "unable to connect to %s", host) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 53a717e..4e66fac 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -8,6 +8,7 @@ import ( "sync/atomic" pb "github.com/converged-computing/rainbow/pkg/api/v1" + "github.com/converged-computing/rainbow/pkg/certs" "github.com/converged-computing/rainbow/pkg/config" "github.com/converged-computing/rainbow/pkg/database" "github.com/converged-computing/rainbow/pkg/graph/backend" @@ -40,6 +41,7 @@ type Server struct { globalToken string db *database.Database host string + certManager *certs.Certificate // graph database handle graph backend.GraphBackend @@ -53,6 +55,7 @@ func NewServer( version, sqliteFile string, cleanup bool, globalToken, host string, + cert *certs.Certificate, ) (*Server, error) { if cfg.Scheduler.Secret == "" { @@ -106,6 +109,7 @@ func NewServer( globalToken: globalToken, selectionAlgorithm: selectAlgo, host: host, + certManager: cert, }, nil } @@ -139,6 +143,7 @@ func (s *Server) Stop() { // Start the server func (s *Server) Start(ctx context.Context, host string) error { + // Create a listener on the specified address. lis, err := net.Listen(protocol, host) if err != nil { @@ -155,8 +160,14 @@ func (s *Server) serve(_ context.Context, lis net.Listener) error { } s.listener = lis - // TODO: should we add grpc.KeepaliveParams here? - s.server = grpc.NewServer() + // If we have a certificate, prepare to load and use it + if !s.certManager.IsEmpty() { + log.Printf("🔐ī¸ adding tls credentials") + tlsCredentials := s.certManager.GetServerCredentials() + s.server = grpc.NewServer(grpc.Creds(tlsCredentials)) + } else { + s.server = grpc.NewServer() + } // This is the main rainbow scheduler service pb.RegisterRainbowSchedulerServer(s.server, s) From b3ac9037ebb3a88d4e56c8baf3a08c3900e0a887 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sat, 20 Apr 2024 00:16:33 -0600 Subject: [PATCH 2/4] save state Signed-off-by: vsoch --- docs/examples/scheduler/rainbow-config.yaml | 2 +- go.mod | 11 ++++----- go.sum | 27 ++++++++------------- hack/client-ext.conf | 2 +- hack/server-ext.conf | 2 +- pkg/certs/certs.go | 21 +++++++++------- pkg/client/client.go | 15 +++++++----- pkg/client/endpoint.go | 8 ++++++ 8 files changed, 47 insertions(+), 41 deletions(-) diff --git a/docs/examples/scheduler/rainbow-config.yaml b/docs/examples/scheduler/rainbow-config.yaml index ada6254..2a26f2e 100644 --- a/docs/examples/scheduler/rainbow-config.yaml +++ b/docs/examples/scheduler/rainbow-config.yaml @@ -8,7 +8,7 @@ scheduler: name: match cluster: name: keebler - secret: 27933478-5c96-4473-bd47-b829b5d0eaf9 + secret: 2ce162e0-7bfe-47ad-85e2-e1e3d6ed87e3 graphdatabase: name: memory host: 127.0.0.1:50051 diff --git a/go.mod b/go.mod index d6127f3..573682b 100644 --- a/go.mod +++ b/go.mod @@ -13,19 +13,18 @@ require ( github.com/mattn/go-sqlite3 v1.14.22 github.com/neo4j/neo4j-go-driver/v5 v5.20.0 github.com/pkg/errors v0.9.1 - google.golang.org/grpc v1.62.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/grpc v1.63.2 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/golang/protobuf v1.5.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 1088156..ec1a711 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,6 @@ github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -31,23 +27,20 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/hack/client-ext.conf b/hack/client-ext.conf index 29ff676..b0956cb 100644 --- a/hack/client-ext.conf +++ b/hack/client-ext.conf @@ -1 +1 @@ -subjectAltName=DNS:*.someclient.com,IP:0.0.0.0 \ No newline at end of file +subjectAltName=DNS:*.localhost,IP:0.0.0.0 \ No newline at end of file diff --git a/hack/server-ext.conf b/hack/server-ext.conf index 9f3add8..3fc8892 100644 --- a/hack/server-ext.conf +++ b/hack/server-ext.conf @@ -1 +1 @@ -subjectAltName=DNS:*localhost,IP:0.0.0.0 \ No newline at end of file +subjectAltName=DNS:0.0.0.0,IP:0.0.0.0 \ No newline at end of file diff --git a/pkg/certs/certs.go b/pkg/certs/certs.go index b93963a..27be2ec 100644 --- a/pkg/certs/certs.go +++ b/pkg/certs/certs.go @@ -18,8 +18,8 @@ type Certificate struct { keyFile string certPool *x509.CertPool - serverCert *tls.Certificate - clientCert *tls.Certificate + serverCert tls.Certificate + clientCert tls.Certificate } // IsEmpty determines if the certificate is empty @@ -66,7 +66,7 @@ func (c *Certificate) GetServerCredentials() credentials.TransportCredentials { // configuration of the certificate what we want to conf := &tls.Config{ - Certificates: []tls.Certificate{*c.serverCert}, + Certificates: []tls.Certificate{c.serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: c.certPool, } @@ -79,12 +79,13 @@ func (c *Certificate) GenerateServerCert() error { if err != nil { return err } - c.serverCert = &serverCert + c.serverCert = serverCert return nil } // NewCertificate generates a client agnostic certificate manager func NewCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) { + // Cut out early if we aren't using TLS cert := Certificate{caCertFile: caCertFile, keyFile: keyFile, certFile: certFile} if cert.IsEmpty() { @@ -100,6 +101,7 @@ func NewCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) return &cert, err } +// NewClientCertificate generates a new client certificate func NewClientCertificate(caCertFile, certFile, keyFile string) (*Certificate, error) { cert, err := NewCertificate(caCertFile, certFile, keyFile) if err != nil || cert.IsEmpty() { @@ -109,13 +111,13 @@ func NewClientCertificate(caCertFile, certFile, keyFile string) (*Certificate, e return cert, err } +// GenerateClientCert generates the client certificate func (c *Certificate) GenerateClientCert() error { - //read client cert clientCert, err := tls.LoadX509KeyPair(c.certFile, c.keyFile) if err != nil { return err } - c.clientCert = &clientCert + c.clientCert = clientCert return nil } @@ -130,10 +132,11 @@ func NewServerCertificate(caCertFile, certFile, keyFile string) (*Certificate, e } // GenerateClientCert generates the client tls certificate -func (c *Certificate) GetClientCredentials() (credentials.TransportCredentials, error) { +func (c *Certificate) GetClientCredentials() credentials.TransportCredentials { config := &tls.Config{ - Certificates: []tls.Certificate{*c.clientCert}, + //InsecureSkipVerify: false, + Certificates: []tls.Certificate{c.clientCert}, RootCAs: c.certPool, } - return credentials.NewTLS(config), nil + return credentials.NewTLS(config) } diff --git a/pkg/client/client.go b/pkg/client/client.go index b0890ed..bf693ca 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -64,18 +64,21 @@ func NewClient(host string, cert *certs.Certificate) (Client, error) { var err error if !cert.IsEmpty() { log.Printf("🔐ī¸ adding tls credentials") - transportCreds, err = cert.GetClientCredentials() - if err != nil { - return nil, err - } + transportCreds = cert.GetClientCredentials() } else { transportCreds = insecure.NewCredentials() } // Set up a connection to the server. creds := grpc.WithTransportCredentials(transportCreds) - conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) - + conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) + + // Something else I was trying + // conn, err := grpc.Dial(c.GetHost(), + // creds, + // grpc.WithBlock(), + // grpc.FailOnNonTempDialError(true), + // ) if err != nil { return nil, errors.Wrapf(err, "unable to connect to %s", host) } diff --git a/pkg/client/endpoint.go b/pkg/client/endpoint.go index e4aaa31..2362878 100644 --- a/pkg/client/endpoint.go +++ b/pkg/client/endpoint.go @@ -15,6 +15,7 @@ import ( "github.com/converged-computing/rainbow/pkg/utils" "github.com/pkg/errors" + "google.golang.org/grpc/connectivity" ts "google.golang.org/protobuf/types/known/timestamppb" ) @@ -185,6 +186,13 @@ func (c *RainbowClient) Register( if secret == "" { return response, errors.New("secret is required") } + + // fmt.Println(c.service) + fmt.Println(c.connection) + fmt.Println(c.connection.GetState()) + fmt.Println(connectivity.Ready) + // return c.service != nil && c.connection != nil && c.connection.GetState() == connectivity.Ready + if !c.Connected() { return response, errors.New("client is not connected") } From 20582faacfa38d0063874eec17e2632de70fe3d1 Mon Sep 17 00:00:00 2001 From: vsoch Date: Fri, 21 Jun 2024 07:50:58 -0600 Subject: [PATCH 3/4] ssl is working for go, likely needs more debugging in Python (and real certs) Signed-off-by: vsoch --- docs/auth.md | 17 +++++++++++++---- docs/examples/scheduler/rainbow-config.yaml | 2 +- hack/client-ext.conf | 2 +- hack/generate-certs.sh | 8 ++++---- hack/server-ext.conf | 2 +- pkg/certs/certs.go | 2 +- pkg/client/client.go | 11 +++-------- pkg/client/endpoint.go | 7 +------ python/v1/README.md | 1 + python/v1/rainbow/auth.py | 18 ++++++++++++++++++ python/v1/rainbow/backends/memory.py | 5 ++--- python/v1/rainbow/client.py | 21 ++++++++++++--------- 12 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 python/v1/rainbow/auth.py diff --git a/docs/auth.md b/docs/auth.md index ae34d43..93cb44b 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -6,7 +6,7 @@ Rainbow currently supports two modes of running: - With certificates (with `--ssl` flag) -Without authentication, just use rainbow as is. +Without authentication, just use rainbow as is. ## With Authentication @@ -67,14 +67,23 @@ With the server running, try doing a register without credentials. make register ``` -WIP not working: https://medium.com/@mertkimyonsen/securing-grpc-connection-with-ssl-tls-certificate-using-go-db3852fe89dd - -It will hang. Likely we can add a timeout to prevent that, but it's not needed for now. Let's add the client credentials and see if we get a different response. +You'll note that it hangs, and doesn't work! Now we can add the client credentials to get a different response: ```bash go run cmd/rainbow/rainbow.go register cluster --cluster-name keebler --nodes-json ./docs/examples/scheduler/cluster-nodes.json \ --config-path ./docs/examples/scheduler/rainbow-config.yaml --save \ --cert /home/vanessa/Desktop/Code/rainbow/bin/certs/client-cert.pem --ca-cert /home/vanessa/Desktop/Code/rainbow/bin/certs/ca-cert.pem --key /home/vanessa/Desktop/Code/rainbow/bin/certs/client-key.pem ``` +```console +2024/06/21 06:51:02 🌈ī¸ starting client (localhost:50051)... +2024/06/21 06:51:02 🔐ī¸ adding tls credentials +2024/06/21 06:51:02 registering cluster: keebler +2024/06/21 06:51:02 status: REGISTER_SUCCESS +2024/06/21 06:51:02 secret: 618c0e98-f4a8-401c-920e-702f6a917c76 +2024/06/21 06:51:02 token: rainbow +2024/06/21 06:51:02 Saving cluster secret to ./docs/examples/scheduler/rainbow-config.yaml +``` + +Note that I followed the tutorial instructions [here](https://medium.com/@mertkimyonsen/securing-grpc-connection-with-ssl-tls-certificate-using-go-db3852fe89dd), although there were a few hairy bits. [home](/README.md#rainbow-scheduler) diff --git a/docs/examples/scheduler/rainbow-config.yaml b/docs/examples/scheduler/rainbow-config.yaml index 2a26f2e..e8516c2 100644 --- a/docs/examples/scheduler/rainbow-config.yaml +++ b/docs/examples/scheduler/rainbow-config.yaml @@ -8,7 +8,7 @@ scheduler: name: match cluster: name: keebler - secret: 2ce162e0-7bfe-47ad-85e2-e1e3d6ed87e3 + secret: 618c0e98-f4a8-401c-920e-702f6a917c76 graphdatabase: name: memory host: 127.0.0.1:50051 diff --git a/hack/client-ext.conf b/hack/client-ext.conf index b0956cb..bfd1f45 100644 --- a/hack/client-ext.conf +++ b/hack/client-ext.conf @@ -1 +1 @@ -subjectAltName=DNS:*.localhost,IP:0.0.0.0 \ No newline at end of file +subjectAltName=DNS:*.localhost,IP:0.0.0.0,DNS:localhost,IP:127.0.0.1,DNS:127.0.0.1 diff --git a/hack/generate-certs.sh b/hack/generate-certs.sh index 75c2a46..fb5a48f 100755 --- a/hack/generate-certs.sh +++ b/hack/generate-certs.sh @@ -9,18 +9,18 @@ fi cd ${root} echo "Building in ${root}" -ls +ls sleep 3 # delete pem file -rm *.pem +rm *.pem # Create CA private key and self-signed certificate # adding -nodes to not encrypt the private key openssl req -x509 -newkey rsa:4096 -nodes -days 365 -keyout ca-key.pem -out ca-cert.pem -subj "/C=US/ST=California/CN=localhost" echo "CA's self-signed certificate" -openssl x509 -in ca-cert.pem -noout -text +openssl x509 -in ca-cert.pem -noout -text # Create Web Server private key and CSR # adding -nodes to not encrypt the private key @@ -45,4 +45,4 @@ openssl x509 -req -in client-req.pem -days 60 -CA ca-cert.pem -CAkey ca-key.pem echo "Client's signed certificate" openssl x509 -in client-cert.pem -noout -text -ls . \ No newline at end of file +ls . diff --git a/hack/server-ext.conf b/hack/server-ext.conf index 3fc8892..9bc3fb3 100644 --- a/hack/server-ext.conf +++ b/hack/server-ext.conf @@ -1 +1 @@ -subjectAltName=DNS:0.0.0.0,IP:0.0.0.0 \ No newline at end of file +subjectAltName=DNS:0.0.0.0,IP:0.0.0.0,DNS:localhost,IP:127.0.0.1,DNS:127.0.0.1 diff --git a/pkg/certs/certs.go b/pkg/certs/certs.go index 27be2ec..12e2433 100644 --- a/pkg/certs/certs.go +++ b/pkg/certs/certs.go @@ -30,7 +30,7 @@ func (c *Certificate) IsEmpty() bool { // Validate all fields are defined func (c *Certificate) Validate() error { if c.IsEmpty() { - return nil + return fmt.Errorf("the certificate is missing all fields") } if c.caCertFile == "" { return fmt.Errorf("the caCertFile is missing") diff --git a/pkg/client/client.go b/pkg/client/client.go index bf693ca..38d3216 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -22,6 +22,7 @@ type RainbowClient struct { host string connection *grpc.ClientConn service pb.RainbowSchedulerClient + creds grpc.DialOption } var _ Client = (*RainbowClient)(nil) @@ -71,14 +72,8 @@ func NewClient(host string, cert *certs.Certificate) (Client, error) { // Set up a connection to the server. creds := grpc.WithTransportCredentials(transportCreds) - conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) - - // Something else I was trying - // conn, err := grpc.Dial(c.GetHost(), - // creds, - // grpc.WithBlock(), - // grpc.FailOnNonTempDialError(true), - // ) + conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) + if err != nil { return nil, errors.Wrapf(err, "unable to connect to %s", host) } diff --git a/pkg/client/endpoint.go b/pkg/client/endpoint.go index 2362878..19f5bc5 100644 --- a/pkg/client/endpoint.go +++ b/pkg/client/endpoint.go @@ -15,7 +15,6 @@ import ( "github.com/converged-computing/rainbow/pkg/utils" "github.com/pkg/errors" - "google.golang.org/grpc/connectivity" ts "google.golang.org/protobuf/types/known/timestamppb" ) @@ -187,15 +186,11 @@ func (c *RainbowClient) Register( return response, errors.New("secret is required") } - // fmt.Println(c.service) - fmt.Println(c.connection) - fmt.Println(c.connection.GetState()) - fmt.Println(connectivity.Ready) // return c.service != nil && c.connection != nil && c.connection.GetState() == connectivity.Ready - if !c.Connected() { return response, errors.New("client is not connected") } + // The cluster nodes file must be defined if clusterNodes == "" { return response, fmt.Errorf("cluster nodes file must be provided with --nodes-json") diff --git a/python/v1/README.md b/python/v1/README.md index 84b2e57..790dffc 100644 --- a/python/v1/README.md +++ b/python/v1/README.md @@ -5,6 +5,7 @@ ![https://github.com/converged-computing/rainbow/raw/main/docs/img/rainbow.png](https://github.com/converged-computing/rainbow/raw/main/docs/img/rainbow.png) This is the rainbow scheduler prototype, specifically Python bindings for a gRPC client. To learn more about rainbow, visit [https://github.com/converged-computing/rainbow](https://github.com/converged-computing/rainbow). +Note that SSL is implemented, but needs to be tested with a production certificate for the Python setup. ## Example diff --git a/python/v1/rainbow/auth.py b/python/v1/rainbow/auth.py new file mode 100644 index 0000000..24aef03 --- /dev/null +++ b/python/v1/rainbow/auth.py @@ -0,0 +1,18 @@ +from contextlib import contextmanager + +import grpc + + +@contextmanager +def grpc_channel(host, use_ssl=False): + """ + Yield a channel, either with or without ssl, and close properly. + """ + if use_ssl: + channel = grpc.secure_channel(host, grpc.ssl_channel_credentials()) + else: + channel = grpc.insecure_channel(host) + try: + yield channel + finally: + channel.close() diff --git a/python/v1/rainbow/backends/memory.py b/python/v1/rainbow/backends/memory.py index 52e8192..32228e5 100644 --- a/python/v1/rainbow/backends/memory.py +++ b/python/v1/rainbow/backends/memory.py @@ -1,5 +1,4 @@ -import grpc - +import rainbow.auth as auth import rainbow.protos.memory_pb2 as memory_pb2 import rainbow.protos.memory_pb2_grpc as memory_pb2_grpc @@ -24,7 +23,7 @@ def satisfies(self, jobspec, matcher="match"): request = memory_pb2.SatisfyRequest(payload=jobspec.to_str(), matcher=matcher) # Host should be set from the database_options from the client - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = memory_pb2_grpc.MemoryGraphStub(channel) response = stub.Satisfy(request) return response diff --git a/python/v1/rainbow/client.py b/python/v1/rainbow/client.py index 7dc25b1..84762a5 100644 --- a/python/v1/rainbow/client.py +++ b/python/v1/rainbow/client.py @@ -5,6 +5,7 @@ import jobspec.core as js import jobspec.core.converter as converter +import rainbow.auth as auth import rainbow.backends as backends import rainbow.config as config import rainbow.defaults as defaults @@ -18,7 +19,7 @@ class RainbowClient: A RainbowClient is able to interact with a Rainbow cluster from Python. """ - def __init__(self, host="localhost:50051", config_file=None, database=None): + def __init__(self, host="localhost:50051", config_file=None, database=None, use_ssl=False): """ Create a new rainbow client to interact with a rainbow cluster. """ @@ -28,6 +29,7 @@ def __init__(self, host="localhost:50051", config_file=None, database=None): # load the graph database backend self.set_database(database) self.load_backend() + self.use_ssl = use_ssl def receive_jobs(self, max_jobs=None): """ @@ -47,7 +49,7 @@ def receive_jobs(self, max_jobs=None): if max_jobs: request.maxJobs = max_jobs - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) response = stub.ReceiveJobs(request) @@ -71,9 +73,10 @@ def receive_jobs(self, max_jobs=None): secret=cluster["secret"], jobids=jobids, ) - with grpc.insecure_channel(self.host) as channel: - stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) - response = stub.AcceptJobs(request) + + stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) + response = stub.AcceptJobs(request) + return jobs def register(self, cluster, secret, cluster_nodes): @@ -97,7 +100,7 @@ def register(self, cluster, secret, cluster_nodes): nodes=nodes, ) - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) response = stub.Register(registerRequest) return response @@ -126,7 +129,7 @@ def update_state(self, cluster, state_data, secret): secret=secret, payload=json.dumps(payload), ) - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) response = stub.UpdateState(request) return response @@ -153,7 +156,7 @@ def register_subsystem(self, cluster, subsystem, secret, nodes): subsystem=subsystem, ) - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) response = stub.RegisterSubsystem(registerRequest) return response @@ -233,7 +236,7 @@ def submit_jobspec( jobspec=jobspec.to_yaml(), ) - with grpc.insecure_channel(self.host) as channel: + with auth.grpc_channel(self.host, self.use_ssl) as channel: stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) response = stub.SubmitJob(submitRequest) From f6b446c67a1c194ffcf1f367bee1fbaa3d56741a Mon Sep 17 00:00:00 2001 From: vsoch Date: Fri, 21 Jun 2024 07:58:59 -0600 Subject: [PATCH 4/4] chore: lint Signed-off-by: vsoch --- cmd/rainbow/config/config.go | 2 -- cmd/rainbow/rainbow.go | 5 +---- docs/examples/scheduler/rainbow-config.yaml | 2 +- pkg/client/client.go | 10 +++------- pkg/client/endpoint.go | 2 -- python/v1/rainbow/client.py | 6 +++--- 6 files changed, 8 insertions(+), 19 deletions(-) diff --git a/cmd/rainbow/config/config.go b/cmd/rainbow/config/config.go index f92770b..93074f3 100644 --- a/cmd/rainbow/config/config.go +++ b/cmd/rainbow/config/config.go @@ -4,7 +4,6 @@ import ( "log" "os" - "github.com/converged-computing/rainbow/pkg/certs" "github.com/converged-computing/rainbow/pkg/config" "gopkg.in/yaml.v3" ) @@ -17,7 +16,6 @@ var ( func RunInit( path string, clusterName, selectAlgo, matchAlgo string, - cert *certs.Certificate, ) error { if path == "" { diff --git a/cmd/rainbow/rainbow.go b/cmd/rainbow/rainbow.go index f17e8b4..79a6b18 100644 --- a/cmd/rainbow/rainbow.go +++ b/cmd/rainbow/rainbow.go @@ -100,9 +100,6 @@ func main() { return } - // TODO: refactor so configuration is generated at this level - // and we don't need to pass all the custom parameters to each client function - // Generate certificate manager cert, err := certs.NewClientCertificate(*caCertFile, *certFile, *keyFile) if err != nil { @@ -111,7 +108,7 @@ func main() { // Config is the only command that doesn't require the client if configCmd.Happened() && configInitCmd.Happened() { - err := config.RunInit(*cfg, *clusterName, *selectAlgo, *matchAlgo, cert) + err := config.RunInit(*cfg, *clusterName, *selectAlgo, *matchAlgo) if err != nil { log.Fatalf("Issue with config: %s\n", err) } diff --git a/docs/examples/scheduler/rainbow-config.yaml b/docs/examples/scheduler/rainbow-config.yaml index e8516c2..490a716 100644 --- a/docs/examples/scheduler/rainbow-config.yaml +++ b/docs/examples/scheduler/rainbow-config.yaml @@ -8,7 +8,7 @@ scheduler: name: match cluster: name: keebler - secret: 618c0e98-f4a8-401c-920e-702f6a917c76 + secret: e823acc8-3cfc-4f98-ab72-8db3f00111a9 graphdatabase: name: memory host: 127.0.0.1:50051 diff --git a/pkg/client/client.go b/pkg/client/client.go index 38d3216..b2957db 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -22,7 +22,6 @@ type RainbowClient struct { host string connection *grpc.ClientConn service pb.RainbowSchedulerClient - creds grpc.DialOption } var _ Client = (*RainbowClient)(nil) @@ -54,12 +53,9 @@ func NewClient(host string, cert *certs.Certificate) (Client, error) { c := &RainbowClient{host: host} // The context allows us to control the timeout - // ctx, cancel := context.WithTimeout(context.TODO(), defaultTimeout) - // defer cancel() + ctx, cancel := context.WithTimeout(context.TODO(), defaultTimeout) + defer cancel() - // Set up a connection to the server. - // creds := grpc.WithTransportCredentials(insecure.NewCredentials()) - // conn, err := grpc.DialContext(ctx, c.GetHost(), creds, grpc.WithBlock()) // Are we using tls? var transportCreds credentials.TransportCredentials var err error @@ -72,7 +68,7 @@ func NewClient(host string, cert *certs.Certificate) (Client, error) { // Set up a connection to the server. creds := grpc.WithTransportCredentials(transportCreds) - conn, err := grpc.Dial(c.GetHost(), creds, grpc.WithBlock()) + conn, err := grpc.DialContext(ctx, c.GetHost(), creds, grpc.WithBlock()) if err != nil { return nil, errors.Wrapf(err, "unable to connect to %s", host) diff --git a/pkg/client/endpoint.go b/pkg/client/endpoint.go index 19f5bc5..9a545de 100644 --- a/pkg/client/endpoint.go +++ b/pkg/client/endpoint.go @@ -185,8 +185,6 @@ func (c *RainbowClient) Register( if secret == "" { return response, errors.New("secret is required") } - - // return c.service != nil && c.connection != nil && c.connection.GetState() == connectivity.Ready if !c.Connected() { return response, errors.New("client is not connected") } diff --git a/python/v1/rainbow/client.py b/python/v1/rainbow/client.py index 84762a5..352d049 100644 --- a/python/v1/rainbow/client.py +++ b/python/v1/rainbow/client.py @@ -74,9 +74,9 @@ def receive_jobs(self, max_jobs=None): jobids=jobids, ) - stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) - response = stub.AcceptJobs(request) - + with auth.grpc_channel(self.host, self.use_ssl) as channel: + stub = rainbow_pb2_grpc.RainbowSchedulerStub(channel) + response = stub.AcceptJobs(request) return jobs def register(self, cluster, secret, cluster_nodes):