From c3f423aab92911890a9ea3abdd4a8a00ba422c45 Mon Sep 17 00:00:00 2001 From: ning1875 <907974064@qq.com> Date: Wed, 24 Feb 2021 15:51:15 +0800 Subject: [PATCH] =?UTF-8?q?*=20[CHANGE]=20=E5=A4=9A=E5=AE=9E=E4=BE=8B?= =?UTF-8?q?=E9=87=87=E9=9B=86=E6=97=B6=EF=BC=8C=E6=9C=80=E7=BB=880?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E6=94=B9=E4=B8=BA=E4=B8=8Dpush=20*=20[CHANGE?= =?UTF-8?q?]=20=E6=8A=8A=E4=B8=80=E4=BA=9B=E5=B8=B8=E8=A7=84info=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E9=99=8D=E7=BA=A7=E6=88=90debug=EF=BC=8C--log.level?= =?UTF-8?q?=3Ddebug=E5=8F=AF=E4=BB=A5=E8=B0=83=E6=95=B4=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 ++ collect/common.go | 2 +- collect/get_node.go | 2 +- collect/get_pod.go | 2 +- collect/kube_apiserver.go | 7 ++++++- collect/kube_controller_manager.go | 6 +++++- collect/kube_coredns.go | 7 ++++++- collect/kube_etcd.go | 7 ++++++- collect/kube_scheduler.go | 7 ++++++- collect/kube_state_metrics.go | 4 ++-- collect/kubelet_cadvisor.go | 2 +- collect/kubelet_node.go | 2 +- collect/push.go | 2 +- k8s-config/daemonSet.yaml | 1 + k8s-config/deployment.yaml | 1 + readme.md | 14 ++++++++++++++ 16 files changed, 55 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e263aa..7a6a20e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## v2.0.5 / 2021-02-24 * [BUGFIX] curl请求采集接口时，http.resp.status_code 非200直接报错返回，避免权限错误引起的解析错误的误导 +* [CHANGE] 多实例采集时，最终0结果改为不push +* [CHANGE] 把一些常规info日志降级成debug，--log.level=debug可以调整日志级别 ## v2.0.4 / 2021-01-28 * [FEATURE] 新增ksm指标计算节点cpu/mem 请求/限制率等指标 diff --git a/collect/common.go b/collect/common.go index 82d92d9..bbe7b41 100644 --- a/collect/common.go +++ b/collect/common.go @@ -478,7 +478,7 @@ func AsyncCurlMetricsAndPush(controlChan chan int, c *config.CommonApiServerConf ml = append(ml, m) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "seq", fmt.Sprintf("%d/%d", index, allNum), "metrics_num", len(ml), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "seq", fmt.Sprintf("%d/%d", index, allNum), "metrics_num", len(ml), "time_took_seconds", time.Since(start).Seconds()) go PushWork(pushServerAddr, tw, ml, logger, funcName) } diff --git a/collect/get_node.go b/collect/get_node.go index 506fa5a..4c25e68 100644 --- a/collect/get_node.go +++ b/collect/get_node.go @@ -49,7 +49,7 @@ func GetServerAddrByGetNode(logger log.Logger, dataMap *HistoryMap) { } } } - level.Info(logger).Log("msg", "server_node_ips_result", + level.Debug(logger).Log("msg", "server_node_ips_result", "num_nodeIps", len(nodeIps), "time_took_seconds", time.Since(start).Seconds(), ) diff --git a/collect/get_pod.go b/collect/get_pod.go index 58e3f37..c730b27 100644 --- a/collect/get_pod.go +++ b/collect/get_pod.go @@ -96,7 +96,7 @@ func GetServerAddrByGetPod(logger log.Logger, dataMap *HistoryMap) { } } - level.Info(logger).Log("msg", "server_pod_ips_result", + level.Debug(logger).Log("msg", "server_pod_ips_result", "num_kubeSchedulerIps", len(kubeSchedulerIps), "num_kubeControllerIps", len(kubeControllerIps), "num_apiServerIps", len(apiServerIps), diff --git a/collect/kube_apiserver.go b/collect/kube_apiserver.go index 0042385..6b7218e 100644 --- a/collect/kube_apiserver.go +++ b/collect/kube_apiserver.go @@ -311,6 +311,11 @@ func DoApiServerCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMa } + if len(metricList) == 0 { + level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName) + return + } + // 开始算quantile newtagsm := map[string]string{ cg.MultiFuncUniqueLabel: funcName, @@ -349,7 +354,7 @@ func DoApiServerCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMa } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kube_controller_manager.go b/collect/kube_controller_manager.go index 4770c45..076d4fe 100644 --- a/collect/kube_controller_manager.go +++ b/collect/kube_controller_manager.go @@ -154,6 +154,10 @@ func DoKubeControllerCollect(cg *config.Config, logger log.Logger, dataMap *Hist metricList = append(metricList, metric) } } + if len(metricList) == 0 { + level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName) + return + } newtagsm := map[string]string{ cg.MultiFuncUniqueLabel: funcName, @@ -173,7 +177,7 @@ func DoKubeControllerCollect(cg *config.Config, logger log.Logger, dataMap *Hist metricList = append(metricList, mm...) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kube_coredns.go b/collect/kube_coredns.go index bb5c0ea..03bb442 100644 --- a/collect/kube_coredns.go +++ b/collect/kube_coredns.go @@ -127,6 +127,11 @@ func DoKubeCoreDnsCollect(cg *config.Config, logger log.Logger, dataMap *History } } + if len(metricList) == 0 { + level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName) + return + } + newtagsm := map[string]string{ cg.MultiFuncUniqueLabel: funcName, } @@ -144,7 +149,7 @@ func DoKubeCoreDnsCollect(cg *config.Config, logger log.Logger, dataMap *History metricList = append(metricList, mm...) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kube_etcd.go b/collect/kube_etcd.go index ecba1f4..56d7c58 100644 --- a/collect/kube_etcd.go +++ b/collect/kube_etcd.go @@ -156,6 +156,11 @@ func DoKubeEtcdCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap } } + if len(metricList) == 0 { + level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName) + return + } + newtagsm := map[string]string{ cg.MultiFuncUniqueLabel: funcName, } @@ -174,7 +179,7 @@ func DoKubeEtcdCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap metricList = append(metricList, mm...) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kube_scheduler.go b/collect/kube_scheduler.go index 467ce9c..fc9a1e1 100644 --- a/collect/kube_scheduler.go +++ b/collect/kube_scheduler.go @@ -278,6 +278,11 @@ func DoKubeSchedulerCollect(cg *config.Config, logger log.Logger, dataMap *Histo } + if len(metricList) == 0 { + level.Error(logger).Log("msg", "CurlTlsMetricsResFinallyEmptyNotPush", "func_name", funcName) + return + } + newtagsm := map[string]string{ cg.MultiFuncUniqueLabel: funcName, } @@ -301,7 +306,7 @@ func DoKubeSchedulerCollect(cg *config.Config, logger log.Logger, dataMap *Histo metricList = append(metricList, mm...) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds()) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kube_state_metrics.go b/collect/kube_state_metrics.go index 445bbf7..278b13f 100644 --- a/collect/kube_state_metrics.go +++ b/collect/kube_state_metrics.go @@ -29,7 +29,7 @@ func DoKubeStatsMetricsCollect(cg *config.Config, logger log.Logger, funcName st level.Error(logger).Log("msg", "DoKubeStatsMetricsCollectEmptyMetricsResult") return } - level.Info(logger).Log("msg", "DoKubeStatsMetricsCollectCurlTlsMetricsApiRes", "resNum", len(metrics)) + level.Debug(logger).Log("msg", "DoKubeStatsMetricsCollectCurlTlsMetricsApiRes", "resNum", len(metrics)) // kube_deployment_labels{deployment="test-server01-deployment", env="prod", instance="10.100.30.139:8080", job="business", label_app="test-server01", //label_dtree_name="bive", label_dtree_nid="200", namespace="default", region="ap-south-1", service="k8s-mon"} @@ -241,7 +241,7 @@ func DoKubeStatsMetricsCollect(cg *config.Config, logger log.Logger, funcName st // pod metricList = PercentComputeForKsm(kube_pod_info_m, kube_node_status_capacity_pods_m, cg.ServerSideNid, "kube_node_pod_num", "node", cg.Step, newtagsm, metricList) - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeStatsC.Addr) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeStatsC.Addr) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kubelet_cadvisor.go b/collect/kubelet_cadvisor.go index 4a93769..dbef09c 100644 --- a/collect/kubelet_cadvisor.go +++ b/collect/kubelet_cadvisor.go @@ -416,7 +416,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap, // disk.bytes.used.percent = container_fs_usage_bytes / container_fs_limit_bytes metricList = CommonComputeFunc(dataMapContainerFsUsage, dataMapContainerFsLimite, metricMapContainerFsUsage, metricList, "disk.bytes.used.percent") - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletC.Addr) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletC.Addr) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, metricList, logger, funcName) } diff --git a/collect/kubelet_node.go b/collect/kubelet_node.go index 4613468..1e5c24d 100644 --- a/collect/kubelet_node.go +++ b/collect/kubelet_node.go @@ -53,7 +53,7 @@ func DoKubeletNodeOnNodeCollect(cg *config.Config, logger log.Logger, dataMap *H ml = append(ml, m) } - level.Info(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletNodeC.Addr) + level.Debug(logger).Log("msg", "DoCollectSuccessfullyReadyToPush", "funcName", funcName, "metrics_num", len(metricList), "time_took_seconds", time.Since(start).Seconds(), "metric_addr", cg.KubeletNodeC.Addr) go PushWork(cg.PushServerAddr, cg.TimeOutSeconds, ml, logger, funcName) diff --git a/collect/push.go b/collect/push.go index 49d29d5..65aae58 100644 --- a/collect/push.go +++ b/collect/push.go @@ -22,7 +22,7 @@ func PushWork(url string, tt int64, metricList []dataobj.MetricValue, logger log return } if resp.StatusCode == 200 { - level.Info(logger).Log("msg", "PushWorkSuccess", "funcName", funcName, "url", url, "metricsNum", len(metricList), "time_took_seconds", time.Since(start).Seconds()) + level.Debug(logger).Log("msg", "PushWorkSuccess", "funcName", funcName, "url", url, "metricsNum", len(metricList), "time_took_seconds", time.Since(start).Seconds()) return } defer resp.Body.Close() diff --git a/k8s-config/daemonSet.yaml b/k8s-config/daemonSet.yaml index 5c3fbcb..e419bbf 100644 --- a/k8s-config/daemonSet.yaml +++ b/k8s-config/daemonSet.yaml @@ -24,6 +24,7 @@ spec: command: - /opt/app/k8s-mon - --config.file=/etc/k8s-mon/k8s-mon.yml + #- --log.level=debug volumeMounts: - mountPath: /etc/k8s-mon name: config-volume diff --git a/k8s-config/deployment.yaml b/k8s-config/deployment.yaml index c2b7825..39b86f5 100644 --- a/k8s-config/deployment.yaml +++ b/k8s-config/deployment.yaml @@ -27,6 +27,7 @@ spec: command: - /opt/app/k8s-mon - --config.file=/etc/k8s-mon/k8s-mon.yml + #- --log.level=debug volumeMounts: - mountPath: /etc/k8s-mon name: config-volume diff --git a/readme.md b/readme.md index 404b751..2eefd6f 100644 --- a/readme.md +++ b/readme.md @@ -213,6 +213,20 @@ append_tags: > 如需服务组件采集多实例时的特征标签 - 修改`k8s-config/configMap_deployment.yaml` 中的 `multi_server_instance_unique_label` 字段 +> 调整日志级别 +- 修改`k8s-config/deployment.yaml` 中的 spec.containers.command 加上`--log.level=debug`即可看到debug日志，日志样例如下 +- 单项数据处理耗时 +```shell script +level=debug ts=2021-02-24T15:47:31.810+08:00 caller=kube_controller_manager.go:180 msg=DoCollectSuccessfullyReadyToPush funcName=kube-controller-manager metrics_num=621 time_took_seconds=0.307592776 +``` +- 单项推送耗时 +```shell script +level=debug ts=2021-02-24T15:47:31.863+08:00 caller=push.go:25 msg=PushWorkSuccess funcName=kube-controller-manager url=http://localhost:2080/api/collector/push metricsNum=621 time_took_seconds=0.053355322 +``` +- 获取pod耗时 +```shell script +level=debug ts=2021-02-24T15:50:01.523+08:00 caller=get_pod.go:99 msg=server_pod_ips_result num_kubeSchedulerIps=1 num_kubeControllerIps=1 num_apiServerIps=1 num_coreDnsIps=2 num_kubeProxyIps=2 num_etcdIps=1 time_took_seconds=0.020384107 +``` ## setup04 启动服务