feat: Enable engine metrics exposing in the agent (#112)

* fix: local metric server should be updated due to k8s version change * feat: enable metric scrape endpoint * chore: enable automaxproc * chore: remove unused file
rakutentech · Oct 1, 2024 · d7d25ad · d7d25ad
1 parent 7e9f6dc
commit d7d25ad
Show file tree

Hide file tree

Showing 9 changed files with 450 additions and 117 deletions.
diff --git a/kubernetes/metricServer.yaml b/kubernetes/metricServer.yaml
@@ -1,19 +1,76 @@
-kind: ClusterRole
+# This is copied from kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
-  name: system:aggregated-metrics-reader
   labels:
-    rbac.authorization.k8s.io/aggregate-to-view: "true"
-    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    k8s-app: metrics-server
     rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: system:aggregated-metrics-reader
 rules:
-- apiGroups: ["metrics.k8s.io"]
-  resources: ["pods"]
-  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/metrics
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  labels:
+    k8s-app: metrics-server
   name: metrics-server:system:auth-delegator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -25,113 +82,122 @@ subjects:
   namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
+kind: ClusterRoleBinding
 metadata:
-  name: metrics-server-auth-reader
-  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: extension-apiserver-authentication-reader
+  kind: ClusterRole
+  name: system:metrics-server
 subjects:
 - kind: ServiceAccount
   name: metrics-server
   namespace: kube-system
 ---
-apiVersion: apiregistration.k8s.io/v1
-kind: APIService
-metadata:
-  name: v1beta1.metrics.k8s.io
-spec:
-  service:
-    name: metrics-server
-    namespace: kube-system
-  group: metrics.k8s.io
-  version: v1beta1
-  insecureSkipTLSVerify: true
-  groupPriorityMinimum: 100
-  versionPriority: 100
----
 apiVersion: v1
-kind: ServiceAccount
+kind: Service
 metadata:
+  labels:
+    k8s-app: metrics-server
   name: metrics-server
   namespace: kube-system
+spec:
+  ports:
+  - name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    k8s-app: metrics-server
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: metrics-server
-  namespace: kube-system
   labels:
     k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
 spec:
   selector:
     matchLabels:
       k8s-app: metrics-server
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
   template:
     metadata:
-      name: metrics-server
       labels:
         k8s-app: metrics-server
     spec:
-      serviceAccountName: metrics-server
-      volumes:
-      # mount in tmp so we can safely use from-scratch images and/or read-only containers
-      - name: tmp-dir
-        emptyDir: {}
       containers:
-      - name: metrics-server
-        image: k8s.gcr.io/metrics-server-amd64:v0.3.1
-        args:
+      - args:
+        - --cert-dir=/tmp
+        - --secure-port=10250
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --kubelet-use-node-status-port
         - --kubelet-insecure-tls
-        - --kubelet-preferred-address-types=InternalIP
-        imagePullPolicy: Always
+        - --metric-resolution=15s
+        image: registry.k8s.io/metrics-server/metrics-server:v0.7.1
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /livez
+            port: https
+            scheme: HTTPS
+          periodSeconds: 10
+        name: metrics-server
+        ports:
+        - containerPort: 10250
+          name: https
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /readyz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 20
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+          seccompProfile:
+            type: RuntimeDefault
         volumeMounts:
-        - name: tmp-dir
-          mountPath: /tmp
-
+        - mountPath: /tmp
+          name: tmp-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: metrics-server
+      volumes:
+      - emptyDir: {}
+        name: tmp-dir
 ---
-apiVersion: v1
-kind: Service
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
 metadata:
-  name: metrics-server
-  namespace: kube-system
   labels:
-    kubernetes.io/name: "Metrics-server"
-spec:
-  selector:
     k8s-app: metrics-server
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: system:metrics-server
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - nodes/stats
-  verbs:
-  - get
-  - list
-  - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system:metrics-server
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:metrics-server
-subjects:
-- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: metrics-server
+    namespace: kube-system
+  version: v1beta1
+  versionPriority: 100
diff --git a/shibuya/controller/collection.go b/shibuya/controller/collection.go
@@ -6,14 +6,14 @@ import (
 	"sync"
 
 	"github.com/rakutentech/shibuya/shibuya/config"
-	controllerModel "github.com/rakutentech/shibuya/shibuya/controller/model"
+	enginesModel "github.com/rakutentech/shibuya/shibuya/engines/model"
 	"github.com/rakutentech/shibuya/shibuya/model"
 	log "github.com/sirupsen/logrus"
 )
 
-func prepareCollection(collection *model.Collection) []*controllerModel.EngineDataConfig {
+func prepareCollection(collection *model.Collection) []*enginesModel.EngineDataConfig {
 	planCount := len(collection.ExecutionPlans)
-	edc := controllerModel.EngineDataConfig{
+	edc := enginesModel.EngineDataConfig{
 		EngineData: map[string]*model.ShibuyaFile{},
 	}
 	engineDataConfigs := edc.DeepCopies(planCount)
@@ -102,7 +102,7 @@ func (c *Controller) TriggerCollection(collection *model.Collection) error {
 			// When all the engines are triggered
 
 			pc := NewPlanController(ep, collection, c.Scheduler)
-			if err := pc.trigger(engineDataConfigs[i]); err != nil {
+			if err := pc.trigger(engineDataConfigs[i], runID); err != nil {
 				errs <- err
 				return
 			}

diff --git a/shibuya/controller/engine.go b/shibuya/controller/engine.go
@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/rakutentech/shibuya/shibuya/config"
-	controllerModel "github.com/rakutentech/shibuya/shibuya/controller/model"
+	enginesModel "github.com/rakutentech/shibuya/shibuya/engines/model"
 	"github.com/rakutentech/shibuya/shibuya/model"
 	sos "github.com/rakutentech/shibuya/shibuya/object_storage"
 	"github.com/rakutentech/shibuya/shibuya/scheduler"
@@ -24,7 +24,7 @@ import (
 )
 
 type shibuyaEngine interface {
-	trigger(edc *controllerModel.EngineDataConfig) error
+	trigger(edc *enginesModel.EngineDataConfig) error
 	deploy(scheduler.EngineScheduler) error
 	subscribe(runID int64) error
 	progress() bool
@@ -76,7 +76,7 @@ type baseEngine struct {
 	*config.ExecutorContainer
 }
 
-func sendTriggerRequest(url string, edc *controllerModel.EngineDataConfig) (*http.Response, error) {
+func sendTriggerRequest(url string, edc *enginesModel.EngineDataConfig) (*http.Response, error) {
 	body := new(bytes.Buffer)
 	json.NewEncoder(body).Encode(&edc)
 	req, _ := http.NewRequest("POST", url, body)
@@ -164,7 +164,7 @@ func (be *baseEngine) deploy(manager scheduler.EngineScheduler) error {
 	return manager.DeployEngine(be.projectID, be.collectionID, be.planID, be.ID, be.ExecutorContainer)
 }
 
-func (be *baseEngine) trigger(edc *controllerModel.EngineDataConfig) error {
+func (be *baseEngine) trigger(edc *enginesModel.EngineDataConfig) error {
 	engineUrl := be.engineUrl
 	base := be.makeBaseUrl()
 	url := fmt.Sprintf(base, engineUrl, "start")

diff --git a/shibuya/controller/plan.go b/shibuya/controller/plan.go
@@ -7,7 +7,7 @@ import (
 	"sync"
 
 	"github.com/rakutentech/shibuya/shibuya/config"
-	controllerModel "github.com/rakutentech/shibuya/shibuya/controller/model"
+	enginesModel "github.com/rakutentech/shibuya/shibuya/engines/model"
 	"github.com/rakutentech/shibuya/shibuya/model"
 	"github.com/rakutentech/shibuya/shibuya/scheduler"
 	_ "github.com/rakutentech/shibuya/shibuya/utils"
@@ -37,7 +37,7 @@ func (pc *PlanController) deploy() error {
 	return nil
 }
 
-func (pc *PlanController) prepare(plan *model.Plan, edc *controllerModel.EngineDataConfig) []*controllerModel.EngineDataConfig {
+func (pc *PlanController) prepare(plan *model.Plan, edc *enginesModel.EngineDataConfig, runID int64) []*enginesModel.EngineDataConfig {
 	edc.Duration = strconv.Itoa(pc.ep.Duration)
 	edc.Concurrency = strconv.Itoa(pc.ep.Concurrency)
 	edc.Rampup = strconv.Itoa(pc.ep.Rampup)
@@ -52,6 +52,8 @@ func (pc *PlanController) prepare(plan *model.Plan, edc *controllerModel.EngineD
 		}
 		// Add test file to all engines
 		engineDataConfigs[i].EngineData[plan.TestFile.Filename] = plan.TestFile
+		engineDataConfigs[i].RunID = runID
+		engineDataConfigs[i].EngineID = i
 		// add all data uploaded in plans. This will override common data if same filename already exists
 		for _, d := range plan.Data {
 			sf := model.ShibuyaFile{
@@ -70,12 +72,12 @@ func (pc *PlanController) prepare(plan *model.Plan, edc *controllerModel.EngineD
 	return engineDataConfigs
 }
 
-func (pc *PlanController) trigger(engineDataConfig *controllerModel.EngineDataConfig) error {
+func (pc *PlanController) trigger(engineDataConfig *enginesModel.EngineDataConfig, runID int64) error {
 	plan, err := model.GetPlan(pc.ep.PlanID)
 	if err != nil {
 		return err
 	}
-	engineDataConfigs := pc.prepare(plan, engineDataConfig)
+	engineDataConfigs := pc.prepare(plan, engineDataConfig, runID)
 	engines, err := generateEnginesWithUrl(pc.ep.Engines, pc.ep.PlanID, pc.collection.ID, pc.collection.ProjectID,
 		JmeterEngineType, pc.scheduler)
 	if err != nil {