From 734c3a64f65949a2ba5fb3a97f24adf0b7719778 Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Tue, 28 Jan 2025 11:53:01 +0100 Subject: [PATCH 1/2] add bootstrap removed event from CEO --- e2echart/e2e-chart-template.html | 6 ++++++ pkg/monitor/monitorapi/types.go | 2 ++ .../operatorloganalyzer/operator_log_scraper.go | 13 ++++++++++++- test/extended/testdata/bindata.go | 6 ++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/e2echart/e2e-chart-template.html b/e2echart/e2e-chart-template.html index 3d95acce5f18..a63b96f7f5da 100644 --- a/e2echart/e2e-chart-template.html +++ b/e2echart/e2e-chart-template.html @@ -101,6 +101,11 @@
Resource
} + function isEtcdBootstrap(eventInterval) { + return eventInterval.source === 'PodLog' && eventInterval.message.reason === "EtcdBootstrap"; + + } + function isPodLog(eventInterval) { if (eventInterval.source === 'PodLog') { return true @@ -500,6 +505,7 @@
Resource
timelineGroups.push({ group: "etcd-leaders", data: [] }) createTimelineData(etcdLeadershipLogsValue, timelineGroups[timelineGroups.length - 1].data, eventIntervals, isEtcdLeadershipAndNotEmpty, regex) + createTimelineData("Bootstrap", timelineGroups[timelineGroups.length - 1].data, eventIntervals, isEtcdBootstrap, regex) timelineGroups.push({group: "cloud-metrics", data: []}) createTimelineData(cloudMetricsValue, timelineGroups[timelineGroups.length - 1].data, eventIntervals, isCloudMetrics, regex) diff --git a/pkg/monitor/monitorapi/types.go b/pkg/monitor/monitorapi/types.go index ffbfa47a950e..149cfda44406 100644 --- a/pkg/monitor/monitorapi/types.go +++ b/pkg/monitor/monitorapi/types.go @@ -262,6 +262,8 @@ const ( ReasonHighGeneration IntervalReason = "HighGeneration" ReasonInvalidGeneration IntervalReason = "GenerationViolation" + + ReasonEtcdBootstrap IntervalReason = "EtcdBootstrap" ) type AnnotationKey string diff --git a/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go b/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go index 102bea51d5c2..aa77b74220c9 100644 --- a/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go +++ b/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" - "github.com/openshift/origin/pkg/monitortests/testframework/watchnamespaces" "strings" "time" + "github.com/openshift/origin/pkg/monitortests/testframework/watchnamespaces" + "github.com/openshift/origin/pkg/monitor" "github.com/openshift/origin/pkg/monitor/monitorapi" "github.com/openshift/origin/pkg/monitortestframework" @@ -207,6 +208,16 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { ). Build(logLine.Instant, logLine.Instant), ) + case strings.Contains(logLine.Line, "Removing bootstrap member") || strings.Contains(logLine.Line, "Successfully removed bootstrap member") || strings.Contains(logLine.Line, "Cluster etcd operator bootstrapped successfully"): // ceo removed bootstrap member + g.recorder.AddIntervals( + monitorapi.NewInterval(monitorapi.SourcePodLog, monitorapi.Info). + Locator(logLine.Locator). + Message(monitorapi.NewMessage(). + Reason(monitorapi.ReasonEtcdBootstrap). + HumanMessage(logLine.Line), + ). + Build(logLine.Instant, logLine.Instant), + ) } } diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index fbc0885f40dd..2dc7d2bf9433 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -53556,6 +53556,11 @@ var _e2echartE2eChartTemplateHtml = []byte(` } + function isEtcdBootstrap(eventInterval) { + return eventInterval.source === 'PodLog' && eventInterval.message.reason === "EtcdBootstrap"; + + } + function isPodLog(eventInterval) { if (eventInterval.source === 'PodLog') { return true @@ -53955,6 +53960,7 @@ var _e2echartE2eChartTemplateHtml = []byte(` timelineGroups.push({ group: "etcd-leaders", data: [] }) createTimelineData(etcdLeadershipLogsValue, timelineGroups[timelineGroups.length - 1].data, eventIntervals, isEtcdLeadershipAndNotEmpty, regex) + createTimelineData("Bootstrap", timelineGroups[timelineGroups.length - 1].data, eventIntervals, isEtcdBootstrap, regex) timelineGroups.push({group: "cloud-metrics", data: []}) createTimelineData(cloudMetricsValue, timelineGroups[timelineGroups.length - 1].data, eventIntervals, isCloudMetrics, regex) From a729dbe6176a42ba0a7eb9583bddd86c3fb05979 Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Wed, 29 Jan 2025 11:29:05 +0100 Subject: [PATCH 2/2] operator log scraper: ensure that events have different from/to time so that spyglass would render them --- .../operatorloganalyzer/operator_log_scraper.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go b/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go index aa77b74220c9..6392d7f601e1 100644 --- a/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go +++ b/pkg/monitortests/testframework/operatorloganalyzer/operator_log_scraper.go @@ -173,7 +173,7 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { Reason(monitorapi.LeaseAcquiringStarted). HumanMessage(logLine.Line), ). - Build(logLine.Instant, logLine.Instant), + Build(logLine.Instant, logLine.Instant.Add(time.Second)), ) case strings.Contains(logLine.Line, "successfully acquired lease") && !strings.Contains(logLine.Line, "Degraded"): // need to exclude lines that re-embed the kube-controller-manager log @@ -184,7 +184,7 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { Reason(monitorapi.LeaseAcquired). HumanMessage(logLine.Line), ). - Build(logLine.Instant, logLine.Instant), + Build(logLine.Instant, logLine.Instant.Add(time.Second)), ) case strings.Contains(logLine.Line, "unable to ApplyStatus for operator") && strings.Contains(logLine.Line, "is invalid"): // apply failures @@ -195,7 +195,7 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { Reason(monitorapi.ReasonBadOperatorApply). HumanMessage(logLine.Line), ). - Build(logLine.Instant, logLine.Instant), + Build(logLine.Instant, logLine.Instant.Add(time.Second)), ) case strings.Contains(logLine.Line, "unable to Apply for operator") && strings.Contains(logLine.Line, "is invalid"): // apply failures @@ -206,7 +206,7 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { Reason(monitorapi.ReasonBadOperatorApply). HumanMessage(logLine.Line), ). - Build(logLine.Instant, logLine.Instant), + Build(logLine.Instant, logLine.Instant.Add(time.Second)), ) case strings.Contains(logLine.Line, "Removing bootstrap member") || strings.Contains(logLine.Line, "Successfully removed bootstrap member") || strings.Contains(logLine.Line, "Cluster etcd operator bootstrapped successfully"): // ceo removed bootstrap member g.recorder.AddIntervals( @@ -216,7 +216,7 @@ func (g operatorLogHandler) HandleLogLine(logLine podaccess.LogLineContent) { Reason(monitorapi.ReasonEtcdBootstrap). HumanMessage(logLine.Line), ). - Build(logLine.Instant, logLine.Instant), + Build(logLine.Instant, logLine.Instant.Add(time.Second)), ) }