diff --git a/cmd/main.go b/cmd/main.go index e2a1c6094c..799e0f5507 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -492,7 +492,7 @@ func main() { Options.BMConfig.S3EndpointURL = newUrl Options.InstallerCacheConfig.CacheDir = filepath.Join(Options.GeneratorConfig.GetWorkingDirectory(), "installercache") - installerCache, err := installercache.New(Options.InstallerCacheConfig, eventsHandler, diskStatsHelper, log) + installerCache, err := installercache.New(Options.InstallerCacheConfig, eventsHandler, metricsManager, diskStatsHelper, log) failOnError(err, "failed to instantiate installercache") generator := generator.New(log, objectHandler, Options.GeneratorConfig, providerRegistry, manifestsApi, eventsHandler, installerCache) diff --git a/internal/ignition/installmanifests_test.go b/internal/ignition/installmanifests_test.go index 4b4ea77732..d9a00b5371 100644 --- a/internal/ignition/installmanifests_test.go +++ b/internal/ignition/installmanifests_test.go @@ -94,6 +94,7 @@ var _ = Describe("Bootstrap Ignition Update", func() { manifestsAPI *manifestsapi.MockManifestsAPI eventsHandler *eventsapi.MockHandler installerCache *installercache.Installers + metricsAPI *metrics.MockAPI ) BeforeEach(func() { @@ -105,12 +106,13 @@ var _ = Describe("Bootstrap Ignition Update", func() { err1 = os.WriteFile(examplePath, []byte(bootstrap1), 0600) Expect(err1).NotTo(HaveOccurred()) ctrl = gomock.NewController(GinkgoT()) + metricsAPI = metrics.NewMockAPI(ctrl) installerCacheConfig := installercache.Config{ CacheDir: filepath.Join(workDir, "some-dir", "installercache"), MaxCapacity: installercache.Size(5), MaxReleaseSize: installercache.Size(5), } - installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) + installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metricsAPI, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) Expect(err).NotTo(HaveOccurred()) mockS3Client = s3wrapper.NewMockAPI(ctrl) manifestsAPI = manifestsapi.NewMockManifestsAPI(ctrl) @@ -262,6 +264,7 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 ctrl *gomock.Controller manifestsAPI *manifestsapi.MockManifestsAPI eventsHandler eventsapi.Handler + metricsAPI *metrics.MockAPI installerCache *installercache.Installers ) @@ -286,12 +289,13 @@ SV4bRR9i0uf+xQ/oYRvugQ25Q7EahO5hJIWRf4aULbk36Zpw3++v2KFnF26zqwB6 ctrl = gomock.NewController(GinkgoT()) manifestsAPI = manifestsapi.NewMockManifestsAPI(ctrl) eventsHandler = eventsapi.NewMockHandler(ctrl) + metricsAPI = metrics.NewMockAPI(ctrl) installerCacheConfig := installercache.Config{ CacheDir: filepath.Join(workDir, "some-dir", "installercache"), MaxCapacity: installercache.Size(5), MaxReleaseSize: installercache.Size(5), } - installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) + installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metricsAPI, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) Expect(err).NotTo(HaveOccurred()) }) @@ -456,6 +460,7 @@ var _ = Describe("createHostIgnitions", func() { workDir string manifestsAPI *manifestsapi.MockManifestsAPI eventsHandler eventsapi.Handler + metricsAPI *metrics.MockAPI installerCache *installercache.Installers ) @@ -476,13 +481,14 @@ var _ = Describe("createHostIgnitions", func() { mockS3Client = s3wrapper.NewMockAPI(ctrl) manifestsAPI = manifestsapi.NewMockManifestsAPI(ctrl) eventsHandler = eventsapi.NewMockHandler(ctrl) + metricsAPI = metrics.NewMockAPI(ctrl) cluster = testCluster() installerCacheConfig := installercache.Config{ CacheDir: filepath.Join(workDir, "some-dir", "installercache"), MaxCapacity: installercache.Size(5), MaxReleaseSize: installercache.Size(5), } - installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) + installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metricsAPI, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) Expect(err).NotTo(HaveOccurred()) }) @@ -1740,6 +1746,7 @@ var _ = Describe("Bare metal host generation", func() { ctrl *gomock.Controller manifestsAPI *manifestsapi.MockManifestsAPI eventsHandler eventsapi.Handler + metricsAPI *metrics.MockAPI installerCache *installercache.Installers ) @@ -1750,13 +1757,14 @@ var _ = Describe("Bare metal host generation", func() { ctrl = gomock.NewController(GinkgoT()) manifestsAPI = manifestsapi.NewMockManifestsAPI(ctrl) eventsHandler = eventsapi.NewMockHandler(ctrl) + metricsAPI = metrics.NewMockAPI(ctrl) installerCacheConfig := installercache.Config{ CacheDir: filepath.Join(workDir, "some-dir", "installercache"), MaxCapacity: installercache.Size(5), MaxReleaseSize: installercache.Size(5), ReleaseFetchRetryInterval: 1 * time.Microsecond, } - installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) + installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metricsAPI, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) Expect(err).NotTo(HaveOccurred()) }) @@ -1850,6 +1858,7 @@ var _ = Describe("Import Cluster TLS Certs for ephemeral installer", func() { ctrl *gomock.Controller manifestsAPI *manifestsapi.MockManifestsAPI eventsHandler eventsapi.Handler + metricsAPI *metrics.MockAPI installerCache *installercache.Installers ) @@ -1881,13 +1890,14 @@ var _ = Describe("Import Cluster TLS Certs for ephemeral installer", func() { ctrl = gomock.NewController(GinkgoT()) manifestsAPI = manifestsapi.NewMockManifestsAPI(ctrl) eventsHandler = eventsapi.NewMockHandler(ctrl) + metricsAPI = metrics.NewMockAPI(ctrl) installerCacheConfig := installercache.Config{ CacheDir: filepath.Join(workDir, "some-dir", "installercache"), MaxCapacity: installercache.Size(5), MaxReleaseSize: installercache.Size(5), ReleaseFetchRetryInterval: 1 * time.Microsecond, } - installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) + installerCache, err = installercache.New(installerCacheConfig, eventsHandler, metricsAPI, metrics.NewOSDiskStatsHelper(logrus.New()), logrus.New()) Expect(err).NotTo(HaveOccurred()) }) diff --git a/internal/installercache/installercache.go b/internal/installercache/installercache.go index 24f9efc5ac..5300be85fd 100644 --- a/internal/installercache/installercache.go +++ b/internal/installercache/installercache.go @@ -36,6 +36,7 @@ type Installers struct { eventsHandler eventsapi.Handler diskStatsHelper metrics.DiskStatsHelper config Config + metricsAPI metrics.API } type Size int64 @@ -116,7 +117,7 @@ func (rl *Release) Cleanup(ctx context.Context) error { } // New constructs an installer cache with a given storage capacity -func New(config Config, eventsHandler eventsapi.Handler, diskStatsHelper metrics.DiskStatsHelper, log logrus.FieldLogger) (*Installers, error) { +func New(config Config, eventsHandler eventsapi.Handler, metricsAPI metrics.API, diskStatsHelper metrics.DiskStatsHelper, log logrus.FieldLogger) (*Installers, error) { if config.MaxCapacity > 0 && config.MaxReleaseSize == 0 { return nil, fmt.Errorf("config.MaxReleaseSize (%d bytes) must not be zero", config.MaxReleaseSize) } @@ -128,6 +129,7 @@ func New(config Config, eventsHandler eventsapi.Handler, diskStatsHelper metrics eventsHandler: eventsHandler, diskStatsHelper: diskStatsHelper, config: config, + metricsAPI: metricsAPI, }, nil } @@ -138,14 +140,20 @@ func (i *Installers) Get(ctx context.Context, releaseID, releaseIDMirror, pullSe for { select { case <-ctx.Done(): - return nil, ctx.Err() + err := ctx.Err() + if err == context.DeadlineExceeded { + i.metricsAPI.InstallerCacheGetReleaseTimeout() + } + return nil, err default: release, err := i.get(releaseID, releaseIDMirror, pullSecret, ocRelease, ocpVersion, clusterID) if err == nil { + i.metricsAPI.InstallerCacheGetReleaseOK() return release, nil } _, isCapacityError := err.(*errorInsufficientCacheCapacity) if !isCapacityError { + i.metricsAPI.InstallerCacheGetReleaseError() return nil, errors.Wrapf(err, "failed to get installer path for release %s", releaseID) } time.Sleep(i.config.ReleaseFetchRetryInterval) @@ -164,6 +172,7 @@ func (i *Installers) getDiskUsageIncludingHardlinks() (uint64, error) { func (i *Installers) extractReleaseIfNeeded(path, releaseID, releaseIDMirror, pullSecret, ocpVersion string, ocRelease oc.Release) (extractDuration float64, cached bool, err error) { _, err = os.Stat(path) if err == nil { + i.metricsAPI.InstallerCacheReleaseCached(releaseID) return 0, true, nil // release was found in the cache } if !os.IsNotExist(err) { @@ -181,6 +190,7 @@ func (i *Installers) extractReleaseIfNeeded(path, releaseID, releaseIDMirror, pu if err != nil { return 0, false, err } + i.metricsAPI.InstallerCacheReleaseExtracted(releaseID) return time.Since(extractStartTime).Seconds(), false, nil } @@ -247,6 +257,7 @@ func (i *Installers) shouldEvict(totalUsed int64) (shouldEvict bool) { // // Locking must be done outside evict() to avoid contentions. func (i *Installers) evict() bool { + i.metricsAPI.InstallerCacheTryEviction() // store the file paths files := NewPriorityQueue(&fileInfo{}) links := make([]*fileInfo, 0) @@ -312,6 +323,7 @@ func (i *Installers) evictFile(filePath string) error { if err != nil { return err } + i.metricsAPI.InstallerCacheReleaseEvicted() // if the parent directory was left empty, // remove it to avoid dangling directories parentDir := path.Dir(filePath) @@ -334,10 +346,11 @@ func (i *Installers) pruneExpiredHardLinks(links []*fileInfo, gracePeriod time.D grace := graceTime.Unix() if finfo.info.ModTime().Unix() < grace { i.log.Infof("attempting to prune hard link %s", finfo.path) - err := os.Remove(finfo.path) - if err != nil { + if err := os.Remove(finfo.path); err != nil { i.log.WithError(err).Errorf("failed to prune hard link %s", finfo.path) + continue } + i.metricsAPI.InstallerCachePrunedHardLink() } } } diff --git a/internal/installercache/installercache_test.go b/internal/installercache/installercache_test.go index 9e285e64d4..61adb6d502 100644 --- a/internal/installercache/installercache_test.go +++ b/internal/installercache/installercache_test.go @@ -71,6 +71,7 @@ var _ = Describe("installer cache", func() { manager *Installers cacheDir string eventsHandler *eventsapi.MockHandler + metricsAPI *metrics.MockAPI ctx context.Context diskStatsHelper metrics.DiskStatsHelper ) @@ -85,17 +86,17 @@ var _ = Describe("installer cache", func() { } BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) diskStatsHelper = metrics.NewOSDiskStatsHelper(logrus.New()) mockRelease = oc.NewMockRelease(ctrl) eventsHandler = eventsapi.NewMockHandler(ctrl) + metricsAPI = metrics.NewMockAPI(ctrl) var err error cacheDir, err = os.MkdirTemp("/tmp", "cacheDir") Expect(err).NotTo(HaveOccurred()) Expect(os.Mkdir(filepath.Join(cacheDir, "quay.io"), 0755)).To(Succeed()) Expect(os.Mkdir(filepath.Join(filepath.Join(cacheDir, "quay.io"), "release-dev"), 0755)).To(Succeed()) - manager, err = New(getInstallerCacheConfig(12, 5), eventsHandler, diskStatsHelper, logrus.New()) + manager, err = New(getInstallerCacheConfig(12, 5), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).NotTo(HaveOccurred()) ctx = context.TODO() }) @@ -138,8 +139,12 @@ var _ = Describe("installer cache", func() { fname := filepath.Join(workdir, releaseID) if !expectCached { mockReleaseCalls(releaseID, version) + metricsAPI.EXPECT().InstallerCacheReleaseExtracted(releaseID).Times(1) } expectEventsSent() + mockReleaseCalls(releaseID, version) + expectEventsSent() + metricsAPI.EXPECT().InstallerCacheGetReleaseOK().Times(1) l, err := manager.Get(ctx, releaseID, "mirror", "pull-secret", mockRelease, version, clusterID) Expect(err).ShouldNot(HaveOccurred()) Expect(l.releaseID).To(Equal(releaseID)) @@ -189,6 +194,11 @@ var _ = Describe("installer cache", func() { runTest := func(t test, manager *Installers) (*Release, error) { expectEventsSent() mockReleaseCalls(t.releaseID, t.version) + metricsAPI.EXPECT().InstallerCacheReleaseCached(t.releaseID).AnyTimes() + metricsAPI.EXPECT().InstallerCacheReleaseExtracted(t.releaseID).AnyTimes() + metricsAPI.EXPECT().InstallerCacheGetReleaseOK().AnyTimes() + metricsAPI.EXPECT().InstallerCacheTryEviction().AnyTimes() + metricsAPI.EXPECT().InstallerCacheReleaseEvicted().AnyTimes() return manager.Get(ctx, t.releaseID, "mirror", "pull-secret", mockRelease, t.version, t.clusterID) } @@ -221,7 +231,7 @@ var _ = Describe("installer cache", func() { // returns the first error encountered or nil if no error encountered. runParallelTest := func(maxCapacity int64, maxReleaseSize int64, tests []test) error { var err error - manager, err = New(getInstallerCacheConfig(maxCapacity, maxReleaseSize), eventsHandler, diskStatsHelper, getLogger()) + manager, err = New(getInstallerCacheConfig(maxCapacity, maxReleaseSize), eventsHandler, metricsAPI, diskStatsHelper, getLogger()) Expect(err).ToNot(HaveOccurred()) var wg sync.WaitGroup var reportedError error @@ -290,30 +300,30 @@ var _ = Describe("installer cache", func() { }) It("Should raise error on construction if max release size is larger than cache and cache is enabled", func() { - _, err := New(getInstallerCacheConfig(5, 10), eventsHandler, diskStatsHelper, logrus.New()) + _, err := New(getInstallerCacheConfig(5, 10), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("config.MaxReleaseSize (10 bytes) must not be greater than config.MaxCapacity (5 bytes)")) }) It("Should raise error on construction if max release size is zero and cache is enabled", func() { - _, err := New(getInstallerCacheConfig(5, 0), eventsHandler, diskStatsHelper, logrus.New()) + _, err := New(getInstallerCacheConfig(5, 0), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("config.MaxReleaseSize (0 bytes) must not be zero")) }) It("Should not raise error on construction if max release size is larger than cache and cache eviction is disabled", func() { - _, err := New(getInstallerCacheConfig(0, 10), eventsHandler, diskStatsHelper, logrus.New()) + _, err := New(getInstallerCacheConfig(0, 10), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).ToNot(HaveOccurred()) }) It("Should not raise error on construction if max release size is zero and cache eviction is disabled", func() { - _, err := New(getInstallerCacheConfig(0, 0), eventsHandler, diskStatsHelper, logrus.New()) + _, err := New(getInstallerCacheConfig(0, 0), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).ToNot(HaveOccurred()) }) It("when cache limit is zero - eviction is skipped", func() { var err error - manager, err = New(getInstallerCacheConfig(0, 5), eventsHandler, diskStatsHelper, logrus.New()) + manager, err = New(getInstallerCacheConfig(0, 5), eventsHandler, metricsAPI, diskStatsHelper, logrus.New()) Expect(err).ToNot(HaveOccurred()) clusterId := strfmt.UUID(uuid.New().String()) r1, _ := testGet("4.8", "4.8.0", clusterId, false) @@ -333,7 +343,10 @@ var _ = Describe("installer cache", func() { clusterId := strfmt.UUID(uuid.New().String()) _, _ = testGet("4.8", "4.8.0", clusterId, false) r2, _ := testGet("4.9", "4.9.0", clusterId, false) + metricsAPI.EXPECT().InstallerCacheReleaseCached("4.8").Times(1) r1, _ := testGet("4.8", "4.8.0", clusterId, true) + metricsAPI.EXPECT().InstallerCacheTryEviction().Times(1) + metricsAPI.EXPECT().InstallerCacheReleaseEvicted().Times(1) r3, _ := testGet("4.10", "4.10.0", clusterId, false) By("verify that the oldest file was deleted") @@ -351,6 +364,8 @@ var _ = Describe("installer cache", func() { clusterId := strfmt.UUID(uuid.New().String()) r1, _ := testGet("4.8", "4.8.0", clusterId, false) r2, _ := testGet("4.9", "4.9.0", clusterId, false) + metricsAPI.EXPECT().InstallerCacheTryEviction().Times(1) + metricsAPI.EXPECT().InstallerCacheReleaseEvicted().Times(1) r3, _ := testGet("4.10", "4.10.0", clusterId, false) By("verify that the oldest file was deleted") @@ -371,6 +386,8 @@ var _ = Describe("installer cache", func() { version := "4.10.0" clusterID := strfmt.UUID(uuid.NewString()) mockReleaseCalls(releaseID, version) + metricsAPI.EXPECT().InstallerCacheReleaseExtracted(releaseID).Times(1) + metricsAPI.EXPECT().InstallerCacheGetReleaseOK().Times(1) l, err := manager.Get(ctx, releaseID, releaseMirrorID, "pull-secret", mockRelease, version, clusterID) Expect(err).ShouldNot(HaveOccurred()) Expect(l.releaseID).To(Equal(releaseID)) @@ -389,7 +406,7 @@ var _ = Describe("installer cache", func() { numberOfLinks := 10 numberOfExpiredLinks := 5 - + metricsAPI.EXPECT().InstallerCachePrunedHardLink().Times(numberOfExpiredLinks) directory, err := os.MkdirTemp("", "testPruneExpiredHardLinks") Expect(err).ToNot(HaveOccurred()) diff --git a/internal/metrics/metricsManager.go b/internal/metrics/metricsManager.go index 6e5111f280..3d1d2b268c 100644 --- a/internal/metrics/metricsManager.go +++ b/internal/metrics/metricsManager.go @@ -39,6 +39,14 @@ const ( counterFilesystemUsagePercentage = "assisted_installer_filesystem_usage_percentage" counterMonitoredHosts = "assisted_installer_monitored_hosts" counterMonitoredClusters = "assisted_installer_monitored_clusters" + counterInstallerCachePrunedHardlink = "assisted_installer_cache_pruned_hardlink" + counterInstallerCacheGetReleaseOK = "assisted_installer_cache_get_release_ok" + counterInstallerCacheGetReleaseTimeout = "assisted_installer_cache_get_release_timeout" + counterInstallerCacheGetReleaseError = "assisted_installer_cache_get_release_error" + counterInstallerCacheReleaseCached = "assisted_installer_cache_get_release_cached" + counterInstallerCacheReleaseExtracted = "assisted_installer_cache_get_release_extracted" + counterInstallerCacheTryEviction = "assisted_installer_cache_try_eviction" + counterInstallerCacheReleaseEvicted = "assisted_installer_cache_release_evicted" ) const ( @@ -61,6 +69,14 @@ const ( counterDescriptionFilesystemUsagePercentage = "The percentage of the filesystem usage by the service" counterDescriptionMonitoredHosts = "Number of hosts monitored by host monitor" counterDescriptionMonitoredClusters = "Number of clusters monitored by cluster monitor" + counterDescriptionInstallerCachePrunedHardlink = "Counts the number of times the installercache pruned a hardlink for being too old" + counterDescriptionInstallerCacheGetReleaseOK = "Counts the number of times that a release was fetched succesfully" + counterDescriptionInstallerCacheGetReleaseTimeout = "Counts the number of times that a release timed out or had the context cancelled" + counterDescriptionInstallerCacheGetReleaseError = "Counts the number of times that a release fetch resulted in error" + counterDescriptionInstallerCacheReleaseCached = "Counts the number of times that a release was found in the cache" + counterDescriptionInstallerCacheReleaseExtracted = "Counts the number of times that a release was extracted" + counterDescriptionInstallerCacheTryEviction = "Counts the number of times that the eviction function was called" + counterDescriptionInstallerCacheReleaseEvicted = "Counts the number of times that a release was evicted" ) const ( @@ -94,6 +110,14 @@ type API interface { FileSystemUsage(usageInPercentage float64) MonitoredHostsCount(monitoredHosts int64) MonitoredClusterCount(monitoredClusters int64) + InstallerCacheReleaseCached(releaseID string) + InstallerCacheReleaseExtracted(releaseID string) + InstallerCacheReleaseEvicted() + InstallerCachePrunedHardLink() + InstallerCacheGetReleaseOK() + InstallerCacheGetReleaseTimeout() + InstallerCacheGetReleaseError() + InstallerCacheTryEviction() } type MetricsManager struct { @@ -119,7 +143,16 @@ type MetricsManager struct { serviceLogicFilesystemUsagePercentage *prometheus.GaugeVec serviceLogicMonitoredHosts *prometheus.GaugeVec serviceLogicMonitoredClusters *prometheus.GaugeVec - collectors []prometheus.Collector + serviceLogicInstallerCachePrunedHardlink *prometheus.CounterVec + serviceLogicInstallerCacheGetReleaseOK *prometheus.CounterVec + serviceLogicInstallerCacheGetReleaseTimeout *prometheus.CounterVec + serviceLogicInstallerCacheGetReleaseError *prometheus.CounterVec + serviceLogicInstallerCacheReleaseCached *prometheus.CounterVec + serviceLogicInstallerCacheReleaseExtracted *prometheus.CounterVec + serviceLogicInstallerCacheTryEviction *prometheus.CounterVec + serviceLogicInstallerCacheReleaseEvicted *prometheus.CounterVec + + collectors []prometheus.Collector } var _ API = &MetricsManager{} @@ -288,6 +321,70 @@ func NewMetricsManager(registry prometheus.Registerer, eventsHandler eventsapi.H Name: counterMonitoredClusters, Help: counterDescriptionMonitoredClusters, }, []string{hosts}), + + serviceLogicInstallerCacheGetReleaseOK: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheGetReleaseOK, + Help: counterDescriptionInstallerCacheGetReleaseOK, + }, []string{}), + + serviceLogicInstallerCachePrunedHardlink: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCachePrunedHardlink, + Help: counterDescriptionInstallerCachePrunedHardlink, + }, []string{}), + + serviceLogicInstallerCacheGetReleaseTimeout: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheGetReleaseTimeout, + Help: counterDescriptionInstallerCacheGetReleaseTimeout, + }, []string{}), + + serviceLogicInstallerCacheGetReleaseError: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheGetReleaseError, + Help: counterDescriptionInstallerCacheGetReleaseError, + }, []string{}), + + serviceLogicInstallerCacheReleaseCached: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheReleaseCached, + Help: counterDescriptionInstallerCacheReleaseCached, + }, []string{}), + + serviceLogicInstallerCacheReleaseExtracted: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheReleaseExtracted, + Help: counterDescriptionInstallerCacheReleaseExtracted, + }, []string{}), + + serviceLogicInstallerCacheTryEviction: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheTryEviction, + Help: counterDescriptionInstallerCacheTryEviction, + }, []string{}), + + serviceLogicInstallerCacheReleaseEvicted: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: counterInstallerCacheReleaseEvicted, + Help: counterDescriptionInstallerCacheReleaseEvicted, + }, []string{}), } m.collectors = append(m.collectors, newDirectoryUsageCollector(metricsManagerConfig.DirectoryUsageMonitorConfig.Directories, diskStatsHelper, log)) @@ -312,6 +409,14 @@ func NewMetricsManager(registry prometheus.Registerer, eventsHandler eventsapi.H m.serviceLogicFilesystemUsagePercentage, m.serviceLogicMonitoredHosts, m.serviceLogicMonitoredClusters, + m.serviceLogicInstallerCachePrunedHardlink, + m.serviceLogicInstallerCacheGetReleaseOK, + m.serviceLogicInstallerCacheGetReleaseTimeout, + m.serviceLogicInstallerCacheGetReleaseError, + m.serviceLogicInstallerCacheReleaseCached, + m.serviceLogicInstallerCacheReleaseExtracted, + m.serviceLogicInstallerCacheTryEviction, + m.serviceLogicInstallerCacheReleaseEvicted, ) for _, collector := range m.collectors { @@ -486,3 +591,35 @@ func (m *MetricsManager) MonitoredClusterCount(monitoredClusters int64) { func bytesToGib(bytes int64) int64 { return bytes / int64(units.GiB) } + +func (m *MetricsManager) InstallerCacheReleaseCached(releaseID string) { + m.serviceLogicInstallerCacheReleaseCached.WithLabelValues(releaseID).Inc() +} + +func (m *MetricsManager) InstallerCacheReleaseExtracted(releaseID string) { + m.serviceLogicInstallerCacheReleaseExtracted.WithLabelValues(releaseID).Inc() +} + +func (m *MetricsManager) InstallerCacheReleaseEvicted() { + m.serviceLogicInstallerCacheReleaseEvicted.WithLabelValues().Inc() +} + +func (m *MetricsManager) InstallerCachePrunedHardLink() { + m.serviceLogicInstallerCachePrunedHardlink.WithLabelValues().Inc() +} + +func (m *MetricsManager) InstallerCacheGetReleaseOK() { + m.serviceLogicInstallerCacheGetReleaseOK.WithLabelValues().Inc() +} + +func (m *MetricsManager) InstallerCacheGetReleaseTimeout() { + m.serviceLogicInstallerCacheGetReleaseTimeout.WithLabelValues().Inc() +} + +func (m *MetricsManager) InstallerCacheGetReleaseError() { + m.serviceLogicInstallerCacheGetReleaseError.WithLabelValues().Inc() +} + +func (m *MetricsManager) InstallerCacheTryEviction() { + m.serviceLogicInstallerCacheTryEviction.WithLabelValues().Inc() +} diff --git a/internal/metrics/mock_metrics_manager_api.go b/internal/metrics/mock_metrics_manager_api.go index 7d6163539e..493f82b832 100644 --- a/internal/metrics/mock_metrics_manager_api.go +++ b/internal/metrics/mock_metrics_manager_api.go @@ -169,6 +169,102 @@ func (mr *MockAPIMockRecorder) InstallationStarted() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallationStarted", reflect.TypeOf((*MockAPI)(nil).InstallationStarted)) } +// InstallerCacheGetReleaseError mocks base method. +func (m *MockAPI) InstallerCacheGetReleaseError() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheGetReleaseError") +} + +// InstallerCacheGetReleaseError indicates an expected call of InstallerCacheGetReleaseError. +func (mr *MockAPIMockRecorder) InstallerCacheGetReleaseError() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheGetReleaseError", reflect.TypeOf((*MockAPI)(nil).InstallerCacheGetReleaseError)) +} + +// InstallerCacheGetReleaseOK mocks base method. +func (m *MockAPI) InstallerCacheGetReleaseOK() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheGetReleaseOK") +} + +// InstallerCacheGetReleaseOK indicates an expected call of InstallerCacheGetReleaseOK. +func (mr *MockAPIMockRecorder) InstallerCacheGetReleaseOK() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheGetReleaseOK", reflect.TypeOf((*MockAPI)(nil).InstallerCacheGetReleaseOK)) +} + +// InstallerCacheGetReleaseTimeout mocks base method. +func (m *MockAPI) InstallerCacheGetReleaseTimeout() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheGetReleaseTimeout") +} + +// InstallerCacheGetReleaseTimeout indicates an expected call of InstallerCacheGetReleaseTimeout. +func (mr *MockAPIMockRecorder) InstallerCacheGetReleaseTimeout() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheGetReleaseTimeout", reflect.TypeOf((*MockAPI)(nil).InstallerCacheGetReleaseTimeout)) +} + +// InstallerCachePrunedHardLink mocks base method. +func (m *MockAPI) InstallerCachePrunedHardLink() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCachePrunedHardLink") +} + +// InstallerCachePrunedHardLink indicates an expected call of InstallerCachePrunedHardLink. +func (mr *MockAPIMockRecorder) InstallerCachePrunedHardLink() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCachePrunedHardLink", reflect.TypeOf((*MockAPI)(nil).InstallerCachePrunedHardLink)) +} + +// InstallerCacheReleaseCached mocks base method. +func (m *MockAPI) InstallerCacheReleaseCached(releaseID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheReleaseCached", releaseID) +} + +// InstallerCacheReleaseCached indicates an expected call of InstallerCacheReleaseCached. +func (mr *MockAPIMockRecorder) InstallerCacheReleaseCached(releaseID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheReleaseCached", reflect.TypeOf((*MockAPI)(nil).InstallerCacheReleaseCached), releaseID) +} + +// InstallerCacheReleaseEvicted mocks base method. +func (m *MockAPI) InstallerCacheReleaseEvicted() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheReleaseEvicted") +} + +// InstallerCacheReleaseEvicted indicates an expected call of InstallerCacheReleaseEvicted. +func (mr *MockAPIMockRecorder) InstallerCacheReleaseEvicted() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheReleaseEvicted", reflect.TypeOf((*MockAPI)(nil).InstallerCacheReleaseEvicted)) +} + +// InstallerCacheReleaseExtracted mocks base method. +func (m *MockAPI) InstallerCacheReleaseExtracted(releaseID string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheReleaseExtracted", releaseID) +} + +// InstallerCacheReleaseExtracted indicates an expected call of InstallerCacheReleaseExtracted. +func (mr *MockAPIMockRecorder) InstallerCacheReleaseExtracted(releaseID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheReleaseExtracted", reflect.TypeOf((*MockAPI)(nil).InstallerCacheReleaseExtracted), releaseID) +} + +// InstallerCacheTryEviction mocks base method. +func (m *MockAPI) InstallerCacheTryEviction() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "InstallerCacheTryEviction") +} + +// InstallerCacheTryEviction indicates an expected call of InstallerCacheTryEviction. +func (mr *MockAPIMockRecorder) InstallerCacheTryEviction() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallerCacheTryEviction", reflect.TypeOf((*MockAPI)(nil).InstallerCacheTryEviction)) +} + // MonitoredClusterCount mocks base method. func (m *MockAPI) MonitoredClusterCount(monitoredClusters int64) { m.ctrl.T.Helper()