Add file system information to each event (#36065)

This includes: * Unix-like: * device * inode * Windows: * idxlo * idxhi * vol * All: * Fingerprint (if the fingerprint mode is enabled)
elastic · Jul 17, 2023 · e4d287f · e4d287f
1 parent 061cb88
commit e4d287f
Show file tree

Hide file tree

Showing 13 changed files with 321 additions and 59 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -348,6 +348,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
 - Add `clean_session` configuration setting for MQTT input.  {pull}35806[16204]
 - Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
+- Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]
 
 *Auditbeat*
    - Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817]
@@ -442,6 +443,3 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 
 
 ==== Known Issues
-
-
-
diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go
@@ -65,7 +65,7 @@ scanner:
 			Op:      loginp.OpCreate,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 5}, // 5 bytes written
+				Info:     testFileInfo{name: basename, size: 5}, // 5 bytes written
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -88,7 +88,7 @@ scanner:
 			Op:      loginp.OpWrite,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 10}, // +5 bytes appended
+				Info:     testFileInfo{name: basename, size: 10}, // +5 bytes appended
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -110,7 +110,7 @@ scanner:
 			Op:      loginp.OpRename,
 			Descriptor: loginp.FileDescriptor{
 				Filename: newFilename,
-				Info:     testFileInfo{path: newBasename, size: 10},
+				Info:     testFileInfo{name: newBasename, size: 10},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -130,7 +130,7 @@ scanner:
 			Op:      loginp.OpTruncate,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 2},
+				Info:     testFileInfo{name: basename, size: 2},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -150,7 +150,7 @@ scanner:
 			Op:      loginp.OpTruncate,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 2},
+				Info:     testFileInfo{name: basename, size: 2},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -169,7 +169,7 @@ scanner:
 			Op:      loginp.OpDelete,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 2},
+				Info:     testFileInfo{name: basename, size: 2},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -207,7 +207,7 @@ scanner:
 			Descriptor: loginp.FileDescriptor{
 				Filename:    filename,
 				Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
-				Info:        testFileInfo{path: basename, size: 1024},
+				Info:        testFileInfo{name: basename, size: 1024},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -238,7 +238,7 @@ scanner:
 			Op:      loginp.OpCreate,
 			Descriptor: loginp.FileDescriptor{
 				Filename: filename,
-				Info:     testFileInfo{path: basename, size: 1024},
+				Info:     testFileInfo{name: basename, size: 1024},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -278,7 +278,7 @@ scanner:
 			Descriptor: loginp.FileDescriptor{
 				Filename:    filename,
 				Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
-				Info:        testFileInfo{path: basename, size: 1024},
+				Info:        testFileInfo{name: basename, size: 1024},
 			},
 		}
 		requireEqualEvents(t, expEvent, e)
@@ -372,35 +372,35 @@ scanner:
 					Filename: normalFilename,
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				undersizedFilename: {
 					Filename: undersizedFilename,
 					Info: testFileInfo{
 						size: sizes[undersizedFilename],
-						path: undersizedBasename,
+						name: undersizedBasename,
 					},
 				},
 				excludedFilename: {
 					Filename: excludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedFilename],
-						path: excludedBasename,
+						name: excludedBasename,
 					},
 				},
 				excludedIncludedFilename: {
 					Filename: excludedIncludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 				travelerSymlinkFilename: {
 					Filename: travelerSymlinkFilename,
 					Info: testFileInfo{
 						size: sizes[travelerFilename],
-						path: travelerSymlinkBasename,
+						name: travelerSymlinkBasename,
 					},
 				},
 			},
@@ -421,28 +421,28 @@ scanner:
 					Filename: normalFilename,
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				undersizedFilename: {
 					Filename: undersizedFilename,
 					Info: testFileInfo{
 						size: sizes[undersizedFilename],
-						path: undersizedBasename,
+						name: undersizedBasename,
 					},
 				},
 				excludedFilename: {
 					Filename: excludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedFilename],
-						path: excludedBasename,
+						name: excludedBasename,
 					},
 				},
 				excludedIncludedFilename: {
 					Filename: excludedIncludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 			},
@@ -464,21 +464,21 @@ scanner:
 					Filename: normalFilename,
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				undersizedFilename: {
 					Filename: undersizedFilename,
 					Info: testFileInfo{
 						size: sizes[undersizedFilename],
-						path: undersizedBasename,
+						name: undersizedBasename,
 					},
 				},
 				travelerSymlinkFilename: {
 					Filename: travelerSymlinkFilename,
 					Info: testFileInfo{
 						size: sizes[travelerFilename],
-						path: travelerSymlinkBasename,
+						name: travelerSymlinkBasename,
 					},
 				},
 			},
@@ -495,14 +495,14 @@ scanner:
 					Filename: normalFilename,
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				undersizedFilename: {
 					Filename: undersizedFilename,
 					Info: testFileInfo{
 						size: sizes[undersizedFilename],
-						path: undersizedBasename,
+						name: undersizedBasename,
 					},
 				},
 			},
@@ -524,7 +524,7 @@ scanner:
 					Filename: excludedIncludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 			},
@@ -541,7 +541,7 @@ scanner:
 					Filename: excludedIncludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 			},
@@ -558,14 +558,14 @@ scanner:
 					Filename: excludedIncludedFilename,
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 				travelerSymlinkFilename: {
 					Filename: travelerSymlinkFilename,
 					Info: testFileInfo{
 						size: sizes[travelerFilename],
-						path: travelerSymlinkBasename,
+						name: travelerSymlinkBasename,
 					},
 				},
 			},
@@ -587,31 +587,31 @@ scanner:
 					Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				excludedFilename: {
 					Filename:    excludedFilename,
 					Fingerprint: "bd151321c3bbdb44185414a1b56b5649a00206dd4792e7230db8904e43987336",
 					Info: testFileInfo{
 						size: sizes[excludedFilename],
-						path: excludedBasename,
+						name: excludedBasename,
 					},
 				},
 				excludedIncludedFilename: {
 					Filename:    excludedIncludedFilename,
 					Fingerprint: "bfdb99a65297062658c26dfcea816d76065df2a2da2594bfd9b96e9e405da1c2",
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 				travelerSymlinkFilename: {
 					Filename:    travelerSymlinkFilename,
 					Fingerprint: "c4058942bffcea08810a072d5966dfa5c06eb79b902bf0011890dd8d22e1a5f8",
 					Info: testFileInfo{
 						size: sizes[travelerFilename],
-						path: travelerSymlinkBasename,
+						name: travelerSymlinkBasename,
 					},
 				},
 			},
@@ -633,7 +633,7 @@ scanner:
 					Fingerprint: "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb",
 					Info: testFileInfo{
 						size: sizes[normalFilename],
-						path: normalBasename,
+						name: normalBasename,
 					},
 				},
 				// undersizedFilename got excluded because of the matching fingerprint
@@ -642,23 +642,23 @@ scanner:
 					Fingerprint: "9c225a1e6a7df9c869499e923565b93937e88382bb9188145f117195cd41dcd1",
 					Info: testFileInfo{
 						size: sizes[excludedFilename],
-						path: excludedBasename,
+						name: excludedBasename,
 					},
 				},
 				excludedIncludedFilename: {
 					Filename:    excludedIncludedFilename,
 					Fingerprint: "7985b2b9750bdd3c76903db408aff3859204d6334279eaf516ecaeb618a218d5",
 					Info: testFileInfo{
 						size: sizes[excludedIncludedFilename],
-						path: excludedIncludedBasename,
+						name: excludedIncludedBasename,
 					},
 				},
 				travelerSymlinkFilename: {
 					Filename:    travelerSymlinkFilename,
 					Fingerprint: "da437600754a8eed6c194b7241b078679551c06c7dc89685a9a71be7829ad7e5",
 					Info: testFileInfo{
 						size: sizes[travelerFilename],
-						path: travelerSymlinkBasename,
+						name: travelerSymlinkBasename,
 					},
 				},
 			},

diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go
@@ -59,7 +59,7 @@ type fileIdentifier interface {
 // fileSource implements the Source interface
 // It is required to identify and manage file sources.
 type fileSource struct {
-	info      loginp.FileDescriptor
+	desc      loginp.FileDescriptor
 	newPath   string
 	oldPath   string
 	truncated bool
@@ -109,7 +109,7 @@ func newINodeDeviceIdentifier(_ *conf.C) (fileIdentifier, error) {
 
 func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource {
 	return fileSource{
-		info:                e.Descriptor,
+		desc:                e.Descriptor,
 		newPath:             e.NewPath,
 		oldPath:             e.OldPath,
 		truncated:           e.Op == loginp.OpTruncate,
@@ -148,7 +148,7 @@ func (p *pathIdentifier) GetSource(e loginp.FSEvent) fileSource {
 		path = e.OldPath
 	}
 	return fileSource{
-		info:                e.Descriptor,
+		desc:                e.Descriptor,
 		newPath:             e.NewPath,
 		oldPath:             e.OldPath,
 		truncated:           e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/identifier_fingerprint.go b/filebeat/input/filestream/identifier_fingerprint.go
@@ -35,7 +35,7 @@ func newFingerprintIdentifier(cfg *conf.C) (fileIdentifier, error) {
 
 func (i *fingerprintIdentifier) GetSource(e loginp.FSEvent) fileSource {
 	return fileSource{
-		info:                e.Descriptor,
+		desc:                e.Descriptor,
 		newPath:             e.NewPath,
 		oldPath:             e.OldPath,
 		truncated:           e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go
@@ -95,7 +95,7 @@ func (i *inodeMarkerIdentifier) markerContents() string {
 func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource {
 	osstate := file.GetOSState(e.Descriptor.Info)
 	return fileSource{
-		info:                e.Descriptor,
+		desc:                e.Descriptor,
 		newPath:             e.NewPath,
 		oldPath:             e.OldPath,
 		truncated:           e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go
@@ -228,7 +228,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo
 
 	r = readfile.NewStripNewline(r, inp.readerConfig.LineTerminator)
 
-	r = readfile.NewFilemeta(r, fs.newPath, offset)
+	r = readfile.NewFilemeta(r, fs.newPath, fs.desc.Info, fs.desc.Fingerprint, offset)
 
 	r = inp.parsers.Create(r)
 

diff --git a/filebeat/input/filestream/prospector_test.go b/filebeat/input/filestream/prospector_test.go
@@ -724,13 +724,13 @@ func TestOnRenameFileIdentity(t *testing.T) {
 }
 
 type testFileInfo struct {
-	path string
+	name string
 	size int64
 	time time.Time
 	sys  interface{}
 }
 
-func (t testFileInfo) Name() string       { return t.path }
+func (t testFileInfo) Name() string       { return t.name }
 func (t testFileInfo) Size() int64        { return t.size }
 func (t testFileInfo) Mode() os.FileMode  { return 0 }
 func (t testFileInfo) ModTime() time.Time { return t.time }