Spanner Deployment (#1561)

* DB_NAME and DB_ENGINE flags * Use noop quota Spanner doesn't have a quota impl * DB_ENGINE=cloud_spanner Inject GOOGLE_APPLICATION_CREDENTIALS * Cloudspanner flags * Set master hold interval * server -v=3 * Set sequencer batch size * pin trillian docker * go get github.com/google/[email protected] * remove mysql sidecar * proper proto copy
google · Jul 3, 2020 · 74d4671 · 74d4671
1 parent 700000d
commit 74d4671
Show file tree

Hide file tree

Showing 13 changed files with 116 additions and 51 deletions.
diff --git a/core/adminserver/admin_server.go b/core/adminserver/admin_server.go
@@ -202,7 +202,7 @@ func privKeyOrGen(ctx context.Context, privKey *any.Any, keygen keys.ProtoGenera
 // - with a set PrivateKey is not nil, otherwise KeySpec is set.
 // - with a tree description of "KT directory %v"
 func treeConfig(treeTemplate *tpb.CreateTreeRequest, privKey *any.Any, directoryID string) *tpb.CreateTreeRequest {
-	config := *treeTemplate
+	config := proto.Clone(treeTemplate).(*tpb.CreateTreeRequest)
 
 	if privKey != nil {
 		config.Tree.PrivateKey = privKey
@@ -215,7 +215,7 @@ func treeConfig(treeTemplate *tpb.CreateTreeRequest, privKey *any.Any, directory
 	if len(directoryID) >= maxDisplayNameLength {
 		config.Tree.DisplayName = directoryID[:maxDisplayNameLength]
 	}
-	return &config
+	return config
 }
 
 // CreateDirectory reachs out to Trillian to produce new trees.

diff --git a/deploy/kubernetes/base/configmap.yaml b/deploy/kubernetes/base/configmap.yaml
@@ -3,4 +3,5 @@ apiVersion: v1
 metadata:
   name: configmap
 data:
-  DB_HOST: db
+  DB_NAME: test:zaphod@tcp(db:3306)/test
+  DB_ENGINE: mysql
diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml
@@ -21,3 +21,20 @@ resources:
 - sequencer-service.yaml
 - server-deployment.yaml
 - server-service.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: gcr.io/key-transparency/init
+  newTag: 15d6d36653d9cce064d2b94ee8d55cab0127fa8b
+- name: gcr.io/key-transparency/keytransparency-monitor
+  newTag: 15d6d36653d9cce064d2b94ee8d55cab0127fa8b
+- name: gcr.io/key-transparency/keytransparency-sequencer
+  newTag: 15d6d36653d9cce064d2b94ee8d55cab0127fa8b
+- name: gcr.io/key-transparency/keytransparency-server
+  newTag: 15d6d36653d9cce064d2b94ee8d55cab0127fa8b
+- name: gcr.io/trillian-opensource-ci/log_server
+  newTag: v1.3.10
+- name: gcr.io/trillian-opensource-ci/log_signer
+  newTag: v1.3.10
+- name: gcr.io/trillian-opensource-ci/map_server
+  newTag: v1.3.10
diff --git a/deploy/kubernetes/base/log-server-deployment.yaml b/deploy/kubernetes/base/log-server-deployment.yaml
@@ -13,10 +13,16 @@ spec:
       labels:
         io.kompose.service: log-server
     spec:
+      volumes: []
       containers:
       - name: trillian-logserver
         args: [
-        "--mysql_uri=test:zaphod@tcp($(DB_HOST):3306)/test",
+        "--mysql_uri=$(DB_NAME)",
+        "--cloudspanner_uri=$(DB_NAME)",
+        "--cloudspanner_dequeue_bucket_fraction=1.0",
+        "--cloudspanner_readonly_staleness=1s",
+        "--storage_system=$(DB_ENGINE)",
+        "--quota_system=noop",
         "--rpc_endpoint=0.0.0.0:8090",
         "--http_endpoint=0.0.0.0:8091",
         "--alsologtostderr"
@@ -34,5 +40,6 @@ spec:
         - containerPort: 8090
         - containerPort: 8091
         resources: {}
+        volumeMounts: []
       restartPolicy: Always
 status: {}
diff --git a/deploy/kubernetes/base/log-signer-deployment.yaml b/deploy/kubernetes/base/log-signer-deployment.yaml
@@ -13,14 +13,21 @@ spec:
       labels:
         io.kompose.service: log-signer
     spec:
+      volumes: []
       containers:
       - name: trillian-logsigner
         args: [
-        "--mysql_uri=test:zaphod@tcp($(DB_HOST):3306)/test",
+        "--mysql_uri=$(DB_NAME)",
+        "--cloudspanner_uri=$(DB_NAME)",
+        "--cloudspanner_dequeue_bucket_fraction=1.0",
+        "--cloudspanner_readonly_staleness=1s",
+        "--storage_system=$(DB_ENGINE)",
+        "--quota_system=noop",
         "--http_endpoint=0.0.0.0:8091",
         "--sequencer_guard_window=0s",
         "--sequencer_interval=1s",
         "--num_sequencers=1",
+        "--master_hold_interval=1h",
         "--batch_size=50",
         "--force_master=true",
         "--alsologtostderr"
@@ -37,5 +44,6 @@ spec:
         ports:
         - containerPort: 8091
         resources: {}
+        volumeMounts: []
       restartPolicy: Always
 status: {}
diff --git a/deploy/kubernetes/base/map-server-deployment.yaml b/deploy/kubernetes/base/map-server-deployment.yaml
@@ -13,10 +13,16 @@ spec:
       labels:
         io.kompose.service: map-server
     spec:
+      volumes: []
       containers:
       - name: trillian-mapserver
         args: [
-        "--mysql_uri=test:zaphod@tcp($(DB_HOST):3306)/test",
+        "--mysql_uri=$(DB_NAME)",
+        "--cloudspanner_uri=$(DB_NAME)",
+        "--storage_system=$(DB_ENGINE)",
+        "--cloudspanner_dequeue_bucket_fraction=1.0",
+        "--cloudspanner_readonly_staleness=1s",
+        "--quota_system=noop",
         "--rpc_endpoint=0.0.0.0:8090",
         "--http_endpoint=0.0.0.0:8091",
         "--single_transaction",
@@ -35,5 +41,6 @@ spec:
         - containerPort: 8090
         - containerPort: 8091
         resources: {}
+        volumeMounts: []
       restartPolicy: Always
 status: {}
diff --git a/deploy/kubernetes/base/sequencer-deployment.yaml b/deploy/kubernetes/base/sequencer-deployment.yaml
@@ -21,13 +21,14 @@ spec:
       - command:
         - /keytransparency-sequencer
         - --force_master
-        - --db=test:zaphod@tcp($(DB_HOST):3306)/test
+        - --db=$(DB_NAME)
+        - --db_engine=$(DB_ENGINE)
         - --addr=0.0.0.0:8080
         - --log-url=log-server:8090
         - --map-url=map-server:8090
         - --tls-key=/run/secrets/tls.key
         - --tls-cert=/run/secrets/tls.crt
-        - --batch-size=500
+        - --batch-size=400
         - --refresh=1s
         - --alsologtostderr
         - --v=5

diff --git a/deploy/kubernetes/base/server-deployment.yaml b/deploy/kubernetes/base/server-deployment.yaml
@@ -21,14 +21,15 @@ spec:
       - command:
         - /keytransparency-server
         - --addr=0.0.0.0:8080
-        - --db=test:zaphod@tcp($(DB_HOST):3306)/test
+        - --db=$(DB_NAME)
+        - --db_engine=$(DB_ENGINE)
         - --log-url=log-server:8090
         - --map-url=map-server:8090
         - --tls-key=/run/secrets/tls.key
         - --tls-cert=/run/secrets/tls.crt
         - --auth-type=insecure-fake
         - --alsologtostderr
-        - --v=5
+        - --v=3
         image: gcr.io/key-transparency/keytransparency-server:latest
         envFrom:
           - configMapRef:

diff --git a/deploy/kubernetes/overlays/gke/cloudspanner-creds.yaml b/deploy/kubernetes/overlays/gke/cloudspanner-creds.yaml
@@ -0,0 +1,14 @@
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: secrets-cloudspanner
+    secret:
+      secretName: cloudspanner
+
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: secrets-cloudspanner
+    mountPath: /secrets/cloudspanner
+    readOnly: true
+
diff --git a/deploy/kubernetes/overlays/gke/configmap.yaml b/deploy/kubernetes/overlays/gke/configmap.yaml
@@ -3,4 +3,7 @@ apiVersion: v1
 metadata:
   name: configmap 
 data:
-  DB_HOST: 127.0.0.1
+  DB_NAME: projects/key-transparency/instances/keytransparency-dev/databases/sandbox
+  DB_ENGINE: cloud_spanner
+  GOOGLE_APPLICATION_CREDENTIALS: /secrets/cloudspanner/credentials.json
+
diff --git a/deploy/kubernetes/overlays/gke/kustomization.yaml b/deploy/kubernetes/overlays/gke/kustomization.yaml
@@ -3,9 +3,12 @@ bases:
 resources:
 - managed-cert.yaml
 patches:
-- path: cloudsql-sidecar.yaml
+- path: cloudspanner-creds.yaml
   target:
     kind: Deployment
+# - path: cloudsql-sidecar.yaml
+#   target:
+#     kind: Deployment
 patchesStrategicMerge:
 - configmap.yaml
 - ingress.yaml

diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/google/keytransparency
 go 1.12
 
 require (
-	cloud.google.com/go/spanner v1.5.2-0.20200519214259-752f788f92e5
+	cloud.google.com/go/spanner v1.7.0
 	github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/go-kit/kit v0.9.0
@@ -14,28 +14,28 @@ require (
 	github.com/google/certificate-transparency-go v1.1.0 // indirect
 	github.com/google/go-cmp v0.5.0
 	github.com/google/tink/go v1.4.0-rc2
-	github.com/google/trillian v1.3.9-0.20200515134453-00fd1983ab83
+	github.com/google/trillian v1.3.10
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/grpc-ecosystem/grpc-gateway v1.14.6
 	github.com/kr/pretty v0.1.0
 	github.com/kylelemons/godebug v1.1.0
 	github.com/pelletier/go-toml v1.6.0 // indirect
-	github.com/prometheus/client_golang v1.7.0
+	github.com/prometheus/client_golang v1.7.1
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spf13/cobra v0.0.7
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.0
 	go.uber.org/atomic v1.5.1 // indirect
 	gocloud.dev v0.20.0
-	golang.org/x/crypto v0.0.0-20200206161412-a0c6ece9d31a
+	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
+	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
 	google.golang.org/api v0.28.0
-	google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482
+	google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5
 	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.24.0
+	google.golang.org/protobuf v1.25.0
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 )