core: added filesystem abstraction layer

include/fluent-bit/flb_file.h

@@ -22,8 +22,81 @@
 #define FLB_FILE_H
 
 #include <fluent-bit/flb_sds.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
-flb_sds_t flb_file_read(const char *path);
-// TODO int flb_file_write(const char *path, flb_sds_t contents);
+#ifdef FLB_SYSTEM_WINDOWS
+#include <fluent-bit/flb_file_win32.h>
+#else
+#include <fluent-bit/flb_file_unix.h>
+#endif
+
+#define FLB_FILE_GLOB_ABORT_ON_ERROR   (((uint64_t) 1) << 0)
+#define FLB_FILE_GLOB_MARK_DIRECTORIES (((uint64_t) 1) << 1)
+#define FLB_FILE_GLOB_DO_NOT_SORT      (((uint64_t) 1) << 2)
+#define FLB_FILE_GLOB_EXPAND_TILDE     (((uint64_t) 1) << 3)
+
+#define FLB_FILE_GLOB_ERROR_SUCCESS          0
+#define FLB_FILE_GLOB_ERROR_ABORTED          1
+#define FLB_FILE_GLOB_ERROR_NO_MEMORY        2
+#define FLB_FILE_GLOB_ERROR_NO_FILE          3
+#define FLB_FILE_GLOB_ERROR_NO_ACCESS        4
+#define FLB_FILE_GLOB_ERROR_NO_MATCHES       5
+#define FLB_FILE_GLOB_ERROR_NO_MORE_RESULTS  6
+#define FLB_FILE_GLOB_ERROR_OVERSIZED_PATH   7
+#define FLB_FILE_GLOB_ERROR_INVALID_ARGUMENT 8
+
+struct flb_file_glob_inner_context;
+
+struct flb_file_glob_context {
+   struct flb_file_glob_inner_context *inner_context;
+   uint64_t                            flags;
+   char                               *path;
+};
+
+struct flb_file_stat {
+   uint64_t  device;
+   uint64_t  inode;
+   uint16_t  mode;
+   int64_t   modification_time;
+   int16_t   hard_link_count;
+   int64_t   size;
+};
+
+int flb_file_glob_start(struct flb_file_glob_context *context,
+                        const char *path,
+                        uint64_t flags);
+
+void flb_file_glob_clean(struct flb_file_glob_context *context);
+
+int flb_file_glob_fetch(struct flb_file_glob_context *context,
+                        char **result);
+
+flb_file_handle flb_file_open(const char *path,
+                              unsigned int flags);
+
+void flb_file_close(flb_file_handle handle);
+
+ssize_t flb_file_read(flb_file_handle handle,
+                      void *output_buffer,
+                      size_t byte_count);
+
+int64_t flb_file_lseek(flb_file_handle handle,
+                       int64_t offset,
+                       int reference_point);
+
+int flb_file_stat(const char *path,
+                  struct flb_file_stat *output_buffer);
+
+int flb_file_lstat(const char *path,
+                   struct flb_file_stat *output_buffer);
+
+int flb_file_fstat(flb_file_handle handle,
+                   struct flb_file_stat *output_buffer);
+
+char *flb_file_get_path(flb_file_handle handle);
+
+char *flb_file_basename(const char *path);
 
+flb_sds_t flb_file_read_contents(const char *path);
 #endif

include/fluent-bit/flb_file_unix.h

@@ -0,0 +1,43 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_FILE_UNIX_H
+#define FLB_FILE_UNIX_H
+
+#define FLB_FILE_IFMT  S_IFMT
+#define FLB_FILE_IFIFO S_IFIFO
+#define FLB_FILE_IFCHR S_IFCHR
+#define FLB_FILE_IFDIR S_IFDIR
+#define FLB_FILE_IFBLK S_IFBLK
+#define FLB_FILE_IFREG S_IFREG
+#define FLB_FILE_IFLNK S_IFLNK
+
+#define FLB_FILE_ISDIR(m)  S_ISDIR(m)
+#define FLB_FILE_ISCHR(m)  S_ISCHR(m)
+#define FLB_FILE_ISFIFO(m) S_ISFIFO(m)
+#define FLB_FILE_ISREG(m)  S_ISREG(m)
+#define FLB_FILE_ISLNK(m)  S_ISLNK(m)
+
+#define FLB_FILE_INVALID_HANDLE  (-1)
+#define FLB_FILE_MAX_PATH_LENGTH PATH_MAX
+
+typedef int flb_file_handle;
+
+#endif

include/fluent-bit/flb_file_win32.h

@@ -0,0 +1,44 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_FILE_WIN32_H
+#define FLB_FILE_WIN32_H
+
+#define FLB_FILE_IFMT  0170000
+#define FLB_FILE_IFIFO 0010000
+#define FLB_FILE_IFCHR 0020000
+#define FLB_FILE_IFDIR 0040000
+#define FLB_FILE_IFBLK 0060000
+#define FLB_FILE_IFREG 0100000
+#define FLB_FILE_IFLNK 0120000
+
+#define FLB_FILE_ISTYPE(m, t) (((m) & FLB_FILE_IFMT) == t)
+#define FLB_FILE_ISDIR(m)     (FLB_FILE_ISTYPE(m, FLB_FILE_IFDIR))
+#define FLB_FILE_ISCHR(m)     (FLB_FILE_ISTYPE(m, FLB_FILE_IFCHR))
+#define FLB_FILE_ISFIFO(m)    (FLB_FILE_ISTYPE(m, FLB_FILE_IFIFO))
+#define FLB_FILE_ISREG(m)     (FLB_FILE_ISTYPE(m, FLB_FILE_IFREG))
+#define FLB_FILE_ISLNK(m)     (FLB_FILE_ISTYPE(m, FLB_FILE_IFLNK))
+
+#define FLB_FILE_INVALID_HANDLE  (INVALID_HANDLE_VALUE)
+#define FLB_FILE_MAX_PATH_LENGTH MAX_PATH
+
+typedef HANDLE flb_file_handle;
+
+#endif

plugins/in_node_exporter_metrics/ne_textfile_linux.c

@@ -143,7 +143,7 @@ static int textfile_update(struct flb_ne *ctx)
     mk_list_foreach(head, &list) {
         entry = mk_list_entry(head, struct flb_slist_entry, _head);
         /* Update metrics from text file */
-        contents = flb_file_read(entry->str);
+        contents = flb_file_read_contents(entry->str);
         if (contents == NULL) {
             flb_plg_debug(ctx->ins, "skip invalid file of prometheus: %s",
                           entry->str);

plugins/out_oracle_log_analytics/oci_logan_conf.c

@@ -103,7 +103,7 @@ static int load_oci_credentials(struct flb_oci_logan *ctx)
     char* key = NULL;
     char* val;
 
-    content = flb_file_read(ctx->config_file_location);
+    content = flb_file_read_contents(ctx->config_file_location);
     if (content == NULL || flb_sds_len(content) == 0)
     {
         return -1;

src/CMakeLists.txt

@@ -82,6 +82,36 @@ set(src
   flb_msgpack_append_message.c
   )
 
+# File access abstraction layer
+if(FLB_SYSTEM_WINDOWS)
+  set(src
+    ${src}
+    flb_file_win32.c
+    )
+else()
+  if(FLB_SYSTEM_LINUX)
+    set(src
+      ${src}
+      flb_file_linux.c
+      )
+  elseif(FLB_SYSTEM_MACOS)
+    set(src
+      ${src}
+      flb_file_macos.c
+      )
+  elseif(FLB_SYSTEM_FREEBSD)
+    set(src
+      ${src}
+      flb_file_freebsd.c
+      )
+  endif()
+
+  set(src
+    ${src}
+    flb_file_unix.c
+    )
+endif()
+
 # Config format
 set(src
   ${src}

src/flb_file.c

@@ -18,14 +18,16 @@
  *  limitations under the License.
  */
 
+#include <fluent-bit/flb_info.h>
 #include <fluent-bit/flb_file.h>
 #include <fluent-bit/flb_log.h>
 #include <fluent-bit/flb_mem.h>
 #include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_str.h>
 
 #include <stdio.h>
 
-flb_sds_t flb_file_read(const char *path)
+flb_sds_t flb_file_read_contents(const char *path)
 {
     long flen;
     FILE *f = NULL;

src/flb_file_freebsd.c

@@ -0,0 +1,63 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_file.h>
+#include <fluent-bit/flb_log.h>
+#include <fluent-bit/flb_mem.h>
+#include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_str.h>
+
+#include <sys/types.h>
+#include <sys/user.h>
+#include <libutil.h>
+#include <stdio.h>
+
+char *flb_file_get_path(flb_file_handle handle)
+{
+    char *buf;
+    struct kinfo_file *file_entries;
+    int file_count;
+    int file_index;
+
+    buf = flb_calloc(sizeof(char), PATH_MAX);
+
+    if (buf == NULL) {
+        flb_errno();
+        return NULL;
+    }
+
+    if ((file_entries = kinfo_getfile(getpid(), &file_count)) == NULL) {
+        flb_free(buf);
+        return NULL;
+    }
+
+    for (file_index=0; file_index < file_count; file_index++) {
+        if (file_entries[file_index].kf_fd == handle) {
+            strncpy(buf, file_entries[file_index].kf_path, PATH_MAX - 1);
+            buf[PATH_MAX - 1] = 0;
+            break;
+        }
+    }
+
+    free(file_entries);
+
+    return buf;
+}

src/flb_file_linux.c

@@ -0,0 +1,62 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_file.h>
+#include <fluent-bit/flb_log.h>
+#include <fluent-bit/flb_mem.h>
+#include <fluent-bit/flb_sds.h>
+#include <fluent-bit/flb_str.h>
+
+#include <stdio.h>
+
+char *flb_file_get_path(flb_file_handle handle)
+{
+    int ret;
+    char *buf;
+    ssize_t s;
+    char tmp[128];
+
+    buf = flb_calloc(sizeof(char), PATH_MAX);
+
+    if (buf == NULL) {
+        flb_errno();
+        return NULL;
+    }
+
+    ret = snprintf(tmp, sizeof(tmp) - 1, "/proc/%i/fd/%i", getpid(), handle);
+    if (ret == -1) {
+        flb_errno();
+        flb_free(buf);
+        return NULL;
+    }
+
+    s = readlink(tmp, buf, PATH_MAX);
+
+    if (s == -1) {
+        flb_free(buf);
+        flb_errno();
+        return NULL;
+    }
+
+    buf[s] = '\0';
+
+    return buf;
+}