fix(kubernetes source)!: Use Pod's UID to exclude Vector's logs

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        kube: [1.14.10, 1.17.2]
+        kube: [1.13.12, 1.14.10, 1.17.2]
     steps:
     - name: Load vector
       uses: actions/download-artifact@v1

config/kubernetes/vector-daemonset.yaml

@@ -84,3 +84,8 @@ spec:
         - name: config-dir
           mountPath: /etc/vector
           readOnly: true
+        env:
+        - name: VECTOR_POD_UID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.uid

docker-compose.yml

@@ -459,7 +459,7 @@ services:
       environment:
         CARGO_TERM_COLOR: always
         RUST_BACKTRACE: full
-        # TEST_LOG: debug
+        TEST_LOG: debug
       volumes:
         - $PWD:$PWD
         - ./target/x86_64-unknown-linux-musl/cargo/registry:/opt/rust/cargo/registry

src/sources/kubernetes/file_source_builder.rs

@@ -28,14 +28,15 @@ impl<'a> FileSourceBuilder<'a> {
         kube_name: &str,
         globals: &GlobalOptions,
         shutdown: ShutdownSignal,
+        vector_pod_uid: &str,
     ) -> crate::Result<(mpsc::Receiver<Event>, Source)> {
         self.file_config.include.extend(
             Self::file_source_include(self.config)?
                 .into_iter()
                 .map(Into::into),
         );
         self.file_config.exclude.extend(
-            Self::file_source_exclude(self.config)
+            Self::file_source_exclude(self.config, vector_pod_uid)
                 .into_iter()
                 .map(Into::into),
         );
@@ -251,7 +252,7 @@ impl<'a> FileSourceBuilder<'a> {
     ///    with include, so exclude isn't necessary.
     /// b) if user has included "kube-system" or "vector*", then that is a sign that user wants
     ///    to log it so excluding it is not valid.
-    fn file_source_exclude(config: &KubernetesConfig) -> Vec<String> {
+    fn file_source_exclude(config: &KubernetesConfig, vector_pod_uid: &str) -> Vec<String> {
         // True if there is no includes
         let no_include = config.include_container_names.is_empty()
             && config.include_namespaces.is_empty()
@@ -266,11 +267,11 @@ impl<'a> FileSourceBuilder<'a> {
             // This is correct, but on best effort basis filtering out of logs from kuberentes system components.
             // More specificly, it will work for all Kubernetes 1.14 and higher, and for some bellow that.
             exclude.push((LOG_DIRECTORY.to_owned() + r"kube-system_*").into());
-
-            // NOTE: for now exclude images with name vector, it's a rough solution, but necessary for now
-            exclude.push((LOG_DIRECTORY.to_owned() + r"*/vector*").into());
         }
 
+        // Always exclude vector
+        exclude.push((LOG_DIRECTORY.to_owned() + &format!("*{}/*", vector_pod_uid)).into());
+
         exclude
     }
 }

src/sources/kubernetes/mod.rs

@@ -20,6 +20,7 @@ use chrono::{DateTime, Utc};
 use futures01::{sync::mpsc, Future, Sink, Stream};
 use serde::{Deserialize, Serialize};
 use snafu::Snafu;
+use std::env::{self, VarError};
 
 // ?NOTE
 // Original proposal: https://github.com/kubernetes/kubernetes/blob/release-1.5/docs/proposals/kubelet-cri-logging.md#proposed-solution
@@ -31,12 +32,21 @@ use snafu::Snafu;
 /// Location in which by Kubernetes CRI, container runtimes are to store logs.
 const LOG_DIRECTORY: &str = r"/var/log/pods/";
 
+/// Enviorment variable through which we are receiving uid of this vector's pod.
+const VECTOR_POD_UID_ENV: &str = "VECTOR_POD_UID";
+
 #[derive(Debug, Snafu)]
 enum BuildError {
     #[snafu(display("To large UID: {:?}", uid))]
     UidToLarge { uid: String },
     #[snafu(display("UID contains illegal characters: {:?}", uid))]
     IllegalCharacterInUid { uid: String },
+    #[snafu(display(
+        "Enviorment variable {}, that must be defined with this Vector's Pod's UID, is {:?}",
+        env,
+        error
+    ))]
+    PodUid { env: &'static str, error: VarError },
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
@@ -65,8 +75,16 @@ impl SourceConfig for KubernetesConfig {
 
         let now = TimeFilter::new();
 
-        let (file_recv, file_source) =
-            file_source_builder::FileSourceBuilder::new(self).build(name, globals, shutdown)?;
+        let vector_pod_uid = env::var(VECTOR_POD_UID_ENV).map_err(|error| BuildError::PodUid {
+            env: VECTOR_POD_UID_ENV,
+            error,
+        })?;
+        let (file_recv, file_source) = file_source_builder::FileSourceBuilder::new(self).build(
+            name,
+            globals,
+            shutdown,
+            &vector_pod_uid,
+        )?;
 
         let mut transform_file = transform_file()?;
         let mut transform_pod_uid = transform_pod_uid()?;

src/sources/kubernetes/test.rs

@@ -134,6 +134,11 @@ spec:
           readOnly: true
         - name: tmp
           mountPath: /tmp/vector/
+        env:
+        - name: VECTOR_POD_UID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.uid
 "#;
 
 static ECHO_YAML: &'static str = r#"
@@ -527,7 +532,7 @@ fn kube_multi_log() {
 
 #[test]
 fn kube_object_uid() {
-    let namespace = "kube-object-uid".to_owned(); //format!("object-uid-{}", Uuid::new_v4());
+    let namespace = format!("object-uid-{}", Uuid::new_v4());
     let message = random_string(300);
     let user_namespace = user_namespace(&namespace);