* [CHANGE] 多实例采集时，最终0结果改为不push

* [CHANGE] 把一些常规info日志降级成debug，--log.level=debug可以调整日志级别
n9e · Feb 24, 2021 · c3f423a · c3f423a
1 parent 5fb4fd5
commit c3f423a
Show file tree

Hide file tree

Showing 16 changed files with 55 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## v2.0.5 / 2021-02-24
 * [BUGFIX] curl请求采集接口时，http.resp.status_code 非200直接报错返回，避免权限错误引起的解析错误的误导
+* [CHANGE] 多实例采集时，最终0结果改为不push
+* [CHANGE] 把一些常规info日志降级成debug，--log.level=debug可以调整日志级别
 
 ## v2.0.4 / 2021-01-28
 * [FEATURE] 新增ksm指标计算节点cpu/mem 请求/限制率等指标

diff --git a/collect/common.go b/collect/common.go
@@ -478,7 +478,7 @@ func AsyncCurlMetricsAndPush(controlChan chan int, c *config.CommonApiServerConf
 		ml = append(ml, m)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "seq", fmt.Sprintf("%d/%d", index, allNum), "metrics_num", len(ml), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "seq", fmt.Sprintf("%d/%d", index, allNum), "metrics_num", len(ml), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(pushServerAddr, tw, ml, logger, funcName)
 
 }

diff --git a/collect/get_node.go b/collect/get_node.go
@@ -49,7 +49,7 @@ func GetServerAddrByGetNode(logger log.Logger, dataMap *HistoryMap) {
 			}
 		}
 	}
-	level.Info(logger).Log("msg", "server_node_ips_result",
+	level.Debug(logger).Log("msg", "server_node_ips_result",
 		"num_nodeIps", len(nodeIps),
 		"time_took_seconds", time.Since(start).Seconds(),
 	)

diff --git a/collect/get_pod.go b/collect/get_pod.go
@@ -96,7 +96,7 @@ func GetServerAddrByGetPod(logger log.Logger, dataMap *HistoryMap) {
 
 		}
 	}
-	level.Info(logger).Log("msg", "server_pod_ips_result",
+	level.Debug(logger).Log("msg", "server_pod_ips_result",
 		"num_kubeSchedulerIps", len(kubeSchedulerIps),
 		"num_kubeControllerIps", len(kubeControllerIps),
 		"num_apiServerIps", len(apiServerIps),

diff --git a/collect/kube_apiserver.go b/collect/kube_apiserver.go
@@ -311,6 +311,11 @@ func DoApiServerCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMa
 
 	}
 
+	if len(metricList) == 0 {
+		level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName)
+		return
+	}
+
 	// 开始算quantile
 	newtagsm := map[string]string{
 		cg.MultiFuncUniqueLabel: funcName,
@@ -349,7 +354,7 @@ func DoApiServerCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMa
 
 	}
 
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 
 }
diff --git a/collect/kube_controller_manager.go b/collect/kube_controller_manager.go
@@ -154,6 +154,10 @@ func DoKubeControllerCollect(cg *config.Config, logger log.Logger, dataMap *Hist
 			metricList = append(metricList, metric)
 		}
 	}
+	if len(metricList) == 0 {
+		level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName)
+		return
+	}
 
 	newtagsm := map[string]string{
 		cg.MultiFuncUniqueLabel: funcName,
@@ -173,7 +177,7 @@ func DoKubeControllerCollect(cg *config.Config, logger log.Logger, dataMap *Hist
 		metricList = append(metricList, mm...)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 
 }
diff --git a/collect/kube_coredns.go b/collect/kube_coredns.go
@@ -127,6 +127,11 @@ func DoKubeCoreDnsCollect(cg *config.Config, logger log.Logger, dataMap *History
 		}
 	}
 
+	if len(metricList) == 0 {
+		level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName)
+		return
+	}
+
 	newtagsm := map[string]string{
 		cg.MultiFuncUniqueLabel: funcName,
 	}
@@ -144,7 +149,7 @@ func DoKubeCoreDnsCollect(cg *config.Config, logger log.Logger, dataMap *History
 		metricList = append(metricList, mm...)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 
 }
diff --git a/collect/kube_etcd.go b/collect/kube_etcd.go
@@ -156,6 +156,11 @@ func DoKubeEtcdCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap
 		}
 	}
 
+	if len(metricList) == 0 {
+		level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName)
+		return
+	}
+
 	newtagsm := map[string]string{
 		cg.MultiFuncUniqueLabel: funcName,
 	}
@@ -174,7 +179,7 @@ func DoKubeEtcdCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap
 		metricList = append(metricList, mm...)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 
 }
diff --git a/collect/kube_scheduler.go b/collect/kube_scheduler.go
@@ -278,6 +278,11 @@ func DoKubeSchedulerCollect(cg *config.Config, logger log.Logger, dataMap *Histo
 
 	}
 
+	if len(metricList) == 0 {
+		level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName)
+		return
+	}
+
 	newtagsm := map[string]string{
 		cg.MultiFuncUniqueLabel: funcName,
 	}
@@ -301,7 +306,7 @@ func DoKubeSchedulerCollect(cg *config.Config, logger log.Logger, dataMap *Histo
 		metricList = append(metricList, mm...)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 
 }
diff --git a/collect/kube_state_metrics.go b/collect/kube_state_metrics.go
@@ -29,7 +29,7 @@ func DoKubeStatsMetricsCollect(cg *config.Config, logger log.Logger, funcName st
 		level.Error(logger).Log("msg", "DoKubeStatsMetricsCollectEmptyMetricsResult")
 		return
 	}
-	level.Info(logger).Log("msg", "DoKubeStatsMetricsCollectCurlTlsMetricsApiRes", "resNum", len(metrics))
+	level.Debug(logger).Log("msg", "DoKubeStatsMetricsCollectCurlTlsMetricsApiRes", "resNum", len(metrics))
 
 	// kube_deployment_labels{deployment="test-server01-deployment", env="prod", instance="10.100.30.139:8080", job="business", label_app="test-server01",
 	//label_dtree_name="bive", label_dtree_nid="200", namespace="default", region="ap-south-1", service="k8s-mon"}
@@ -241,7 +241,7 @@ func DoKubeStatsMetricsCollect(cg *config.Config, logger log.Logger, funcName st
 	// pod
 	metricList = PercentComputeForKsm(kube_pod_info_m, kube_node_status_capacity_pods_m, cg.ServerSideNid, "kube_node_pod_num", "node", cg.Step, newtagsm, metricList)
 
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeStatsC.Addr)
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeStatsC.Addr)
 
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 }
diff --git a/collect/kubelet_cadvisor.go b/collect/kubelet_cadvisor.go
@@ -416,7 +416,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
 	// disk.bytes.used.percent = container_fs_usage_bytes / container_fs_limit_bytes
 	metricList = CommonComputeFunc(dataMapContainerFsUsage, dataMapContainerFsLimite, metricMapContainerFsUsage, metricList, "disk.bytes.used.percent")
 
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletC.Addr)
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletC.Addr)
 
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName)
 }

diff --git a/collect/kubelet_node.go b/collect/kubelet_node.go
@@ -53,7 +53,7 @@ func DoKubeletNodeOnNodeCollect(cg *config.Config, logger log.Logger, dataMap *H
 		ml = append(ml, m)
 
 	}
-	level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletNodeC.Addr)
+	level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletNodeC.Addr)
 
 	go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, ml, logger, funcName)
 

diff --git a/collect/push.go b/collect/push.go
@@ -22,7 +22,7 @@ func PushWork(url string, tt int64, metricList []dataobj.MetricValue, logger log
 			return
 		}
 		if resp.StatusCode == 200 {
-			level.Info(logger).Log("msg", "PushWorkSuccess", "funcName", funcName, "url", url, "metricsNum", len(metricList), "time_took_seconds", time.Since(start).Seconds())
+			level.Debug(logger).Log("msg", "PushWorkSuccess", "funcName", funcName, "url", url, "metricsNum", len(metricList), "time_took_seconds", time.Since(start).Seconds())
 			return
 		}
 		defer resp.Body.Close()

diff --git a/k8s-config/daemonSet.yaml b/k8s-config/daemonSet.yaml
@@ -24,6 +24,7 @@ spec:
           command:
             - /opt/app/k8s-mon
             - --config.file=/etc/k8s-mon/k8s-mon.yml
+            #- --log.level=debug
           volumeMounts:
             - mountPath: /etc/k8s-mon
               name: config-volume

diff --git a/k8s-config/deployment.yaml b/k8s-config/deployment.yaml
@@ -27,6 +27,7 @@ spec:
           command:
             - /opt/app/k8s-mon
             - --config.file=/etc/k8s-mon/k8s-mon.yml
+            #- --log.level=debug
           volumeMounts:
             - mountPath: /etc/k8s-mon
               name: config-volume

diff --git a/readme.md b/readme.md
@@ -213,6 +213,20 @@ append_tags:
 > 如需服务组件采集多实例时的特征标签
 - 修改`k8s-config/configMap_deployment.yaml` 中的 `multi_server_instance_unique_label` 字段
 
+> 调整日志级别
+- 修改`k8s-config/deployment.yaml` 中的 spec.containers.command 加上`--log.level=debug`即可看到debug日志，日志样例如下
+- 单项数据处理耗时
+```shell script
+level=debug ts=2021-02-24T15:47:31.810+08:00 caller=kube_controller_manager.go:180 msg=DoCollectSuccessfullyReadyToPush funcName=kube-controller-manager metrics_num=621 time_took_seconds=0.307592776
+```
+- 单项推送耗时
+```shell script
+level=debug ts=2021-02-24T15:47:31.863+08:00 caller=push.go:25 msg=PushWorkSuccess funcName=kube-controller-manager url=http://localhost:2080/api/collector/push metricsNum=621 time_took_seconds=0.053355322
+```
+- 获取pod耗时
+```shell script
+level=debug ts=2021-02-24T15:50:01.523+08:00 caller=get_pod.go:99 msg=server_pod_ips_result num_kubeSchedulerIps=1 num_kubeControllerIps=1 num_apiServerIps=1 num_coreDnsIps=2 num_kubeProxyIps=2 num_etcdIps=1 time_took_seconds=0.020384107
+``` 
 
 ## setup04 启动服务