From b481143bff40d709a7cd7ade4e78db41a47f4507 Mon Sep 17 00:00:00 2001 From: Denis Date: Mon, 17 Jul 2023 20:50:42 +0200 Subject: [PATCH] Add benchmarks for scanning files (#36078) So, we can compare how much slower the fingerprinting mode against using `device` and `inode` in the scanner. --- filebeat/input/filestream/fswatch_test.go | 57 +++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go index f6dabbaa053..2a30dd8b3cd 100644 --- a/filebeat/input/filestream/fswatch_test.go +++ b/filebeat/input/filestream/fswatch_test.go @@ -19,6 +19,7 @@ package filestream import ( "context" + "fmt" "os" "path/filepath" "strings" @@ -693,6 +694,62 @@ scanner: }) } +const benchmarkFileCount = 1000 + +func BenchmarkGetFiles(b *testing.B) { + dir := b.TempDir() + basenameFormat := "file-%d.log" + + for i := 0; i < benchmarkFileCount; i++ { + filename := filepath.Join(dir, fmt.Sprintf(basenameFormat, i)) + content := fmt.Sprintf("content-%d\n", i) + err := os.WriteFile(filename, []byte(strings.Repeat(content, 1024)), 0777) + require.NoError(b, err) + } + + s := fileScanner{ + paths: []string{filepath.Join(dir, "*.log")}, + cfg: fileScannerConfig{ + Fingerprint: fingerprintConfig{ + Enabled: false, + }, + }, + } + + for i := 0; i < b.N; i++ { + files := s.GetFiles() + require.Len(b, files, benchmarkFileCount) + } +} + +func BenchmarkGetFilesWithFingerprint(b *testing.B) { + dir := b.TempDir() + basenameFormat := "file-%d.log" + + for i := 0; i < benchmarkFileCount; i++ { + filename := filepath.Join(dir, fmt.Sprintf(basenameFormat, i)) + content := fmt.Sprintf("content-%d\n", i) + err := os.WriteFile(filename, []byte(strings.Repeat(content, 1024)), 0777) + require.NoError(b, err) + } + + s := fileScanner{ + paths: []string{filepath.Join(dir, "*.log")}, + cfg: fileScannerConfig{ + Fingerprint: fingerprintConfig{ + Enabled: true, + Offset: 0, + Length: 1024, + }, + }, + } + + for i := 0; i < b.N; i++ { + files := s.GetFiles() + require.Len(b, files, benchmarkFileCount) + } +} + func createWatcherWithConfig(t *testing.T, paths []string, cfgStr string) loginp.FSWatcher { cfg, err := conf.NewConfigWithYAML([]byte(cfgStr), cfgStr) require.NoError(t, err)