Skip to content

Commit

Permalink
* [BUGFIX] hold点/预聚合所使用的共享mapdataMap.Map改为go-cache ，用来做gc，避免pod滚动…
Browse files Browse the repository at this point in the history 
…后旧的数据没有删除导致内存不回收

* [CHANGE] 编译时传入version，便于打印版本信息
  • Loading branch information
@ning1875
ning1875 committed Mar 30, 2021
1 parent c3f423a commit a39862c
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 78 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## v2.0.7 / 2021-03-30
* [BUGFIX] hold点/预聚合所使用的共享map`dataMap.Map`改为`go-cache` ，用来做gc，避免pod滚动后旧的数据没有删除导致内存不回收
* [CHANGE] 编译时传入version，便于打印版本信息

## v2.0.5 / 2021-02-24
* [BUGFIX] curl请求采集接口时，http.resp.status_code 非200直接报错返回，避免权限错误引起的解析错误的误导
* [CHANGE] 多实例采集时，最终0结果改为不push
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ COPY ./go.mod ./
COPY ./go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o server -ldflags "-X 'github.com/prometheus/common/version.BuildUser=root@n9e' -X 'github.com/prometheus/common/version.BuildDate=`date`' "
RUN CGO_ENABLED=0 go build -o server -ldflags "-X 'github.com/prometheus/common/version.BuildUser=root@n9e' -X 'github.com/prometheus/common/version.BuildDate=`date`' -X 'github.com/prometheus/common/version.Version=`cat VERSION`'"
#FROM scratch as runner
FROM busybox as runner
COPY --from=builder /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0
2.0.7
8 changes: 5 additions & 3 deletions collect/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"crypto/md5"
"errors"
"fmt"
"github.com/patrickmn/go-cache"
"io"
"io/ioutil"
"math"
Expand Down Expand Up @@ -205,15 +206,16 @@ func histogramDeltaWork(dataMap *HistoryMap, bucketM map[float64]float64, newtag

s.UpdateCounterStat(thisCounterStats)
mapKey := funcName + sepStr + newMetricName + sepStr + fmt.Sprintf("%f", up)
obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)
if !loaded {
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)
continue
}
dataHis := obj.(*CommonCounterHis)
dataHis.UpdateCounterStat(thisCounterStats)
dataRate := dataHis.DeltaCounter()

dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
newM[up] = dataRate

}
Expand Down Expand Up @@ -433,7 +435,7 @@ func GetServerSideAddr(cg *config.CommonApiServerConfig, logger log.Logger, data

return
}
obj, loaded := dataMap.Map.Load(funcName)
obj, loaded := dataMap.Map.Get(funcName)
if !loaded {
level.Error(logger).Log("msg", "GetServerAddrByGetPodErrorNoValue", "funcName:", funcName)
return
Expand Down
3 changes: 2 additions & 1 deletion collect/get_node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package collect

import (
"context"
"github.com/patrickmn/go-cache"
"time"

"github.com/go-kit/kit/log"
Expand Down Expand Up @@ -54,7 +55,7 @@ func GetServerAddrByGetNode(logger log.Logger, dataMap *HistoryMap) {
"time_took_seconds", time.Since(start).Seconds(),
)
if len(nodeIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_KUBELET_NODE, nodeIps)
dataMap.Map.Set(kconfig.FUNCNAME_KUBELET_NODE, nodeIps, cache.DefaultExpiration)
}

}
13 changes: 7 additions & 6 deletions collect/get_pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package collect

import (
"context"
"github.com/patrickmn/go-cache"
"time"

"github.com/go-kit/kit/log"
Expand Down Expand Up @@ -106,22 +107,22 @@ func GetServerAddrByGetPod(logger log.Logger, dataMap *HistoryMap) {
"time_took_seconds", time.Since(start).Seconds(),
)
if len(coreDnsIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_COREDNS, coreDnsIps)
dataMap.Map.Set(kconfig.FUNCNAME_COREDNS, coreDnsIps, cache.DefaultExpiration)
}
if len(apiServerIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_APISERVER, apiServerIps)
dataMap.Map.Set(kconfig.FUNCNAME_APISERVER, apiServerIps, cache.DefaultExpiration)
}
if len(etcdIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_ETCD, etcdIps)
dataMap.Map.Set(kconfig.FUNCNAME_ETCD, etcdIps, cache.DefaultExpiration)
}
if len(kubeSchedulerIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_KUBESCHEDULER, kubeSchedulerIps)
dataMap.Map.Set(kconfig.FUNCNAME_KUBESCHEDULER, kubeSchedulerIps, cache.DefaultExpiration)
}
if len(kubeControllerIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_KUBECONTROLLER, kubeControllerIps)
dataMap.Map.Set(kconfig.FUNCNAME_KUBECONTROLLER, kubeControllerIps, cache.DefaultExpiration)
}
if len(kubeProxyIps) > 0 {
dataMap.Map.Store(kconfig.FUNCNAME_KUBEPROXY, kubeProxyIps)
dataMap.Map.Set(kconfig.FUNCNAME_KUBEPROXY, kubeProxyIps, cache.DefaultExpiration)
}

}
30 changes: 20 additions & 10 deletions collect/kubelet_cadvisor.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package collect

import (
"github.com/patrickmn/go-cache"
"strings"
"time"

Expand Down Expand Up @@ -198,10 +199,13 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

s.UpdateCounterStat(thisCounterStats)

obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
//obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)
if !loaded {
// 说明第一次
//level.Info(logger).Log("msg", "MapDataNotFound", "metric_name", "container_cpu_usage_seconds_total", "mapKey", mapKey)
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)

continue
} else {
//level.Info(logger).Log("msg", "MapDataGet", "metric_name", "container_cpu_usage_seconds_total", "mapKey", mapKey)
Expand All @@ -213,7 +217,8 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
metric.Metric = "cpu.util"

metricMapContainerCpuUsage[cUniqueKey] = metric
dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
//dataMap.Map.Store(mapKey, dataHis)
newM := metric
newM.Metric = "cpu.cores.occupy"
metricList = append(metricList, newM)
Expand All @@ -225,8 +230,9 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

s.UpdateCounterStat(thisCounterStats)

obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)
if !loaded {
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)
continue
} else {
dataHis := obj.(*CommonCounterHis)
Expand All @@ -236,7 +242,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
metric.Metric = "cpu.user"

metricMapContainerCpuUser[cUniqueKey] = metric
dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
// 这里continue的目的是，metirc已经被改名为cpu.user做计算了
// 而且原始点也没有必要上报了，不需要下面在append 到list中
continue
Expand All @@ -247,10 +253,12 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

s.UpdateCounterStat(thisCounterStats)

obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)

if !loaded {
// 说明第一次
//level.Info(logger).Log("msg", "MapDataNotFound", "metric_name", "container_cpu_system_seconds_total", "mapKey", mapKey)
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)
continue
} else {
//level.Info(logger).Log("msg", "MapDataGet", "metric_name", "container_cpu_system_seconds_total", "mapKey", mapKey)
Expand All @@ -260,7 +268,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
dataMapContainerCpuSys[cUniqueKey] = dataRate
metric.Metric = "cpu.sys"
metricMapContainerCpuSys[cUniqueKey] = metric
dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
continue
}

Expand All @@ -274,8 +282,9 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

s.UpdateCounterStat(thisCounterStats)

obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)
if !loaded {
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)
continue
} else {
//level.Info(logger).Log("msg", "MapDataGet", "metric_name", "container_cpu_system_seconds_total", "mapKey", mapKey)
Expand All @@ -285,7 +294,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,
dataMapContainerCfsPeriods[cUniqueKey] = dataRate

metricMapContainerCfsPeriods[cUniqueKey] = metric
dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
// rename
metric.Metric = "cpu.periods"

Expand All @@ -297,10 +306,11 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

s.UpdateCounterStat(thisCounterStats)

obj, loaded := dataMap.Map.LoadOrStore(mapKey, s)
obj, loaded := dataMap.Map.Get(mapKey)
if !loaded {
// 说明第一次
//level.Info(logger).Log("msg", "MapDataNotFound", "metric_name", "container_cpu_system_seconds_total", "mapKey", mapKey)
dataMap.Map.Set(mapKey, s, cache.DefaultExpiration)
continue
} else {
//level.Info(logger).Log("msg", "MapDataGet", "metric_name", "container_cpu_system_seconds_total", "mapKey", mapKey)
Expand All @@ -310,7 +320,7 @@ func DoKubeletCollect(cg *config.Config, logger log.Logger, dataMap *HistoryMap,

dataMapContainerCfsThrottledPeriods[cUniqueKey] = dataRate
metricMapContainerCfsThrottledPeriods[cUniqueKey] = metric
dataMap.Map.Store(mapKey, dataHis)
dataMap.Map.Set(mapKey, dataHis, cache.DefaultExpiration)
// rename
metric.Metric = "cpu.throttled.periods"
}
Expand Down
11 changes: 8 additions & 3 deletions collect/stats.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package collect

import (
"github.com/patrickmn/go-cache"
"sync"
"time"
)

type HistoryMap struct {
Map sync.Map
//Map sync.Map
Map *cache.Cache
//Cache *cache.Cache
}

type CounterStats struct {
Expand All @@ -19,8 +23,9 @@ type CommonCounterHis struct {
}

func NewHistoryMap() *HistoryMap {
m := sync.Map{}
return &HistoryMap{Map: m}
//m := sync.Map{}
c := cache.New(5*time.Minute, 10*time.Minute)
return &HistoryMap{Map: c}

}

Expand Down
35 changes: 29 additions & 6 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,46 @@ module github.com/n9e/k8s-mon
go 1.15

require (
github.com/Microsoft/go-winio v0.4.16 // indirect
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
github.com/containerd/containerd v1.4.3 // indirect
github.com/didi/nightingale v0.0.0-20201018021739-93c35fd0ec44
github.com/docker/docker v20.10.2+incompatible
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.3+incompatible
github.com/go-kit/kit v0.10.0
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/cadvisor v0.38.7
github.com/kr/text v0.2.0 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/oklog/run v1.1.0
github.com/open-falcon/falcon-plus v0.2.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pelletier/go-toml v1.7.0 // indirect
github.com/prometheus/client_golang v1.9.0 // indirect
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.15.0
github.com/prometheus/common v0.17.0
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.7.0 // indirect
github.com/toolkits/pkg v1.1.3
github.com/toolkits/time v0.0.0-20160524122720-c274716e8d7f // indirect
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd // indirect
golang.org/x/text v0.3.5 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect
google.golang.org/grpc v1.34.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gotest.tools/v3 v3.0.3 // indirect
k8s.io/api v0.20.1
k8s.io/apimachinery v0.20.1
k8s.io/client-go v0.0.0-20210106050432-9761a13537eb
k8s.io/klog v1.0.0 // indirect
k8s.io/api v0.20.2
k8s.io/apimachinery v0.20.2
k8s.io/client-go v0.20.2
k8s.io/klog/v2 v2.5.0 // indirect
)
Loading

0 comments on commit a39862c

Please sign in to comment.