release v1.91.4

.gitignore

@@ -98,7 +98,8 @@ WinFsp/
 bin/
 
 .vscode/
-
+dist
+configure~
 #
 # Local variables:
 # tab-width: 4

README-CN.md

@@ -27,6 +27,7 @@ ossfs 基于s3fs 构建，具有s3fs 的全部功能。主要功能包括：
 * 默认使用single-part拷贝以提升大文件的重名性能
 * 可选符号链接信息保存在对象的元数据里
 * 可选开启目录读取优化模式，该模式下会忽略如下文件信息，atime/ctime, uid/gid 和 permissions
+* 可选开启直读模式，该模式下，读取文件时，数据下载到内存而非磁盘，再从内存中读取，从而提升顺序读性能
 
 ### 安装
 

README.md

@@ -26,7 +26,8 @@ The ossfs is built based on s3fs and has all the features of s3fs. Main features
 In addition to the above features, ossfs also has its own features.
 * renames via server-side single-part copy to improve large files renaming performance
 * optional saves the symbolic link target in object user metadata
-* optional improve readdir perfermence by ignoring metadata-atime/ctime, uid/gid, and permissions
+* optional improve readdir perfermance by ignoring metadata-atime/ctime, uid/gid, and permissions
+* optional improve sequential-read performance by downloading file data to and reading from the memory instead of the disk
 
 ### Precompiled installer
 

configure.ac

@@ -20,7 +20,7 @@
 dnl Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.59)
-AC_INIT(ossfs, 1.91.3)
+AC_INIT(ossfs, 1.91.4)
 AC_CONFIG_HEADER([config.h])
 
 AC_CANONICAL_SYSTEM
@@ -320,6 +320,11 @@ AC_COMPILE_IFELSE(
   ]
 )
 
+dnl ----------------------------------------------
+dnl dl library
+dnl ----------------------------------------------
+AC_CHECK_LIB([dl], [dlopen, dlclose, dlerror, dlsym], [], [AC_MSG_ERROR([Could not found dlopen, dlclose, dlerror and dlsym])])
+
 dnl ----------------------------------------------
 dnl build date
 dnl ----------------------------------------------

doc/man/ossfs.1.in

@@ -238,6 +238,15 @@ payloads. This can reduce CPU overhead to transfers.
 \fB\-o\fR ram_role (default is no IAM role)
 This option requires the IAM role name or "auto". If you specify "auto", ossfs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto".
 .TP
+\fB\-o\fR credlib (default=\"\" which means disabled)
+Specifies the shared library that handles the credentials containing the authentication token.
+If this option is specified, the specified credential and token processing provided by the shared library ant will be performed instead of the built-in credential processing.
+This option cannot be specified with passwd_file, use_session_token, and ram_role option.
+.TP
+\fB\-o\fR credlib_opts (default=\"\" which means disabled)
+Specifies the options to pass when the shared library specified in credlib is loaded and then initialized.
+For the string specified in this option, specify the string defined by the shared library.
+.TP
 \fB\-o\fR use_xattr (default is not handling the extended attribute)
 Enable to handle the extended attribute (xattrs).
 If you set this option, you can use the extended attribute.

scripts/build-pkg.py

@@ -39,9 +39,13 @@
     'anolisos7.0:test':'ossfs-anolisos7.0:test',
     'anolisos8.0:dev':'ossfs-anolisos8.0:dev',
     'anolisos8.0:test':'ossfs-anolisos8.0:test',
+    'alinux2:dev':'ossfs-alinux2:dev',
+    'alinux2:test':'ossfs-alinux2:test',
+    'alinux3:dev':'ossfs-alinux3:dev',
+    'alinux3:test':'ossfs-alinux3:test',
 }
 
-os_list = ['centos7.0', 'centos8.0', 'ubuntu14.04', 'ubuntu16.04', 'ubuntu18.04', 'ubuntu20.04', 'ubuntu22.04', 'anolisos7.0', 'anolisos8.0']
+os_list = ['ubuntu14.04', 'ubuntu16.04', 'ubuntu18.04', 'ubuntu20.04', 'ubuntu22.04', 'centos7.0', 'centos8.0', 'alinux2', 'alinux3']
 working_dir = '/tmp/ossfs'
 dest_dir = '/var/ossfs'
 ossfs_source_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -282,6 +286,35 @@ def command_test_package_anolisos(os_name):
     command_test_package(f)
     f.close()
 
+def command_build_package_alinux(os_name):
+    """
+    Generate the build package script running in docker container
+    """
+    cmd_dir = os.path.join(working_dir, 'command')
+    install_dir = '/tmp/ossfs_install'
+    f = open(os.path.join(cmd_dir, 'build_package_%s.sh'%os_name), 'w')
+    f.write('#!/bin/bash\n')
+    command_build_package(f, install_dir)
+    f.write('fpm -s dir -t rpm -n ossfs -v %s -C %s -p ossfs_VERSION_%s_ARCH.rpm -d "fuse >= 2.8.4" -d "fuse-libs >= 2.8.4" -d "libcurl >= 7.0" -d "libxml2 >= 2.6" -d "openssl-libs >= 0.9"\n' % (ossfs_version, install_dir, os_name))
+    f.close()
+
+def command_test_package_alinux(os_name):
+    """
+    Generate the test package script running in docker container
+    """
+    pkg_list = glob.glob(os.path.join(working_dir, 'package/*%s*'%os_name))
+    if not pkg_list:
+        raise RuntimeError("Can not found %s package! May be build fail?"%os_name)
+    pkg = ntpath.basename(pkg_list[0])
+    cmd_dir = os.path.join(working_dir, 'command')
+    test_dir = os.path.join(dest_dir, 'source/test')
+    f = open(os.path.join(cmd_dir, 'test_package_%s.sh'%os_name), 'w')
+    f.write('#!/bin/bash\n')
+    f.write('yum -y update\n')
+    f.write('yum -y localinstall %s/package/%s --nogpgcheck\n' % (dest_dir, pkg))
+    command_test_package(f)
+    f.close()
+
 def build_docker_image():
     #ubuntu:14.04
     exec_cmd('docker pull ubuntu:14.04')
@@ -310,23 +343,23 @@ def build_docker_image():
 
     #centos:7.x
     exec_cmd('docker pull centos:centos7')
-    exec_cmd('docker tag centos:centos7 ossfs-centos7.0:test')
+    exec_cmd('docker build -t ossfs-centos7.0:test %s/scripts/docker-file/centos/7.x'%ossfs_source_dir)
     exec_cmd('docker build -t ossfs-centos7.0:dev %s/scripts/docker-file/centos/7.x'%ossfs_source_dir)
 
     #centos:8.x
     exec_cmd('docker pull centos:centos8')
     exec_cmd('docker build -t ossfs-centos8.0:dev %s/scripts/docker-file/centos/8.x'%ossfs_source_dir)
     exec_cmd('docker build -t ossfs-centos8.0:test %s/scripts/docker-file/centos/8.x'%ossfs_source_dir)
 
-    #anolisos:7.x
-    exec_cmd('docker pull openanolis/anolisos:7.9-x86_64')
-    exec_cmd('docker tag openanolis/anolisos:7.9-x86_64 ossfs-anolisos7.0:test')
-    exec_cmd('docker build -t ossfs-anolisos7.0:dev %s/scripts/docker-file/anolisos/7.x'%ossfs_source_dir)
+    #alibaba clound linux 2, anolisos:7.x
+    exec_cmd('docker pull alibaba-cloud-linux-2-registry.cn-hangzhou.cr.aliyuncs.com/alinux2/alinux2:latest')
+    exec_cmd('docker tag alibaba-cloud-linux-2-registry.cn-hangzhou.cr.aliyuncs.com/alinux2/alinux2:latest ossfs-alinux2:test')
+    exec_cmd('docker build -t ossfs-alinux2:dev %s/scripts/docker-file/alinux/2'%ossfs_source_dir)
 
-    #centos:8.x
-    exec_cmd('docker pull openanolis/anolisos:8.6')
-    exec_cmd('docker tag openanolis/anolisos:8.6 ossfs-anolisos8.0:test')
-    exec_cmd('docker build -t ossfs-anolisos8.0:dev %s/scripts/docker-file/anolisos/8.x'%ossfs_source_dir)
+    #alibaba cloud linux 3, anolisos: 8.x
+    exec_cmd('docker pull alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3:latest')
+    exec_cmd('docker tag alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3:latest ossfs-alinux3:test')
+    exec_cmd('docker build -t ossfs-alinux3:dev %s/scripts/docker-file/alinux/3'%ossfs_source_dir)
     pass
 
 def build_package():
@@ -416,6 +449,23 @@ def build_package():
             command_test_package_anolisos(os_name)
             container_name = 'ossfs_%s'%random_string(5)
             docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_%s.sh' % (dest_dir, os_name))
+        elif os_name.startswith('alinux'):
+            # build package
+            print("=============================")
+            print("build %s package ..." % os_name)
+            print("=============================")
+            command_build_package_alinux(os_name)
+            container_name = 'ossfs_%s'%random_string(5)
+            docker_run(container_name, dev_image, volumes, '/bin/bash %s/command/build_package_%s.sh' % (dest_dir, os_name))
+
+            # test package
+            print("============================")
+            print("test %s package ..." % os_name)
+            print("============================")
+            command_test_package_alinux(os_name)
+            container_name = 'ossfs_%s'%random_string(5)
+            docker_run(container_name, test_image, volumes, '/bin/bash %s/command/test_package_%s.sh' % (dest_dir, os_name))
+
 
 if __name__ == '__main__':
     build_docker_image()

scripts/docker-file/alinux/2/Dockerfile

@@ -0,0 +1,12 @@
+FROM alibaba-cloud-linux-2-registry.cn-hangzhou.cr.aliyuncs.com/alinux2/alinux2:latest
+
+
+RUN yum update upgrade -y \
+  && yum install -y gcc gcc-c++ make cmake curl rpm-build\
+  && yum install -y automake gcc-c++ git libcurl-devel libxml2-devel fuse-devel openssl-devel \
+  && curl -SL "https://cache.ruby-china.com/pub/ruby/ruby-2.6.5.tar.gz" -o /opt/ruby-2.6.5.tar.gz\
+  && cd /opt/ && tar zxvf ruby-2.6.5.tar.gz && cd ruby-2.6.5 \
+  && ./configure && make -j10 && make install \
+  && gem sources -a http://mirrors.aliyun.com/rubygems/ -r https://rubygems.org/ \
+  && gem install dotenv -v 2.8.1 \
+  && gem install fpm

scripts/docker-file/alinux/3/Dockerfile

@@ -0,0 +1,11 @@
+FROM alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3:latest
+
+RUN yum update -y 
+RUN yum install -y gcc gcc-c++ make cmake curl rpm-build
+RUN yum install -y automake gcc-c++ git libcurl-devel libxml2-devel fuse-devel openssl-devel \
+  && curl -SL "https://cache.ruby-china.com/pub/ruby/ruby-2.6.5.tar.gz" -o /opt/ruby-2.6.5.tar.gz\
+  && cd /opt/ && tar zxvf ruby-2.6.5.tar.gz && cd ruby-2.6.5 \
+  && ./configure && make -j10 && make install \
+  && gem sources -a http://mirrors.aliyun.com/rubygems/ -r https://rubygems.org/ \
+  && gem install dotenv -v 2.8.1 \
+  && gem install fpm

scripts/docker-file/centos/7.x/Dockerfile

@@ -1,7 +1,9 @@
 FROM centos:centos7
 
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
 RUN yum update upgrade -y \
-    && yum install -y gcc gcc-c++ make cmake curl rpm-build\
+    && yum install -y gcc gcc-c++ make cmake curl rpm-build \
     && yum install -y automake gcc-c++ git libcurl-devel libxml2-devel fuse-devel openssl-devel \
     && curl -SL "https://cache.ruby-china.com/pub/ruby/ruby-2.6.5.tar.gz" -o /opt/ruby-2.6.5.tar.gz\
     && cd /opt/ && tar zxvf ruby-2.6.5.tar.gz && cd ruby-2.6.5 \

scripts/docker-file/ubuntu/18.04/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:18.04
 
 RUN    apt-get update \
     && apt-get install -y software-properties-common
- 
+
 RUN    add-apt-repository ppa:brightbox/ruby-ng \
     && apt-get update \
     && apt-get install -y automake autotools-dev g++ git libcurl4-gnutls-dev curl\

scripts/docker-file/ubuntu/20.04/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN    apt-get update \
     && apt-get install -y software-properties-common
- 
+
 RUN    add-apt-repository ppa:brightbox/ruby-ng \
     && apt-get update \
     && apt-get install -y automake autotools-dev g++ git libcurl4-gnutls-dev curl\

src/common.h

@@ -50,6 +50,13 @@ extern std::string    mount_prefix;
 extern std::string    endpoint;
 extern std::string    cipher_suites;
 extern std::string    instance_name;
+extern std::string    region;
+extern std::string    cloudbox_id;
+
+//-------------------------------------------------------------------
+// For weak attribute
+//-------------------------------------------------------------------
+#define	S3FS_FUNCATTR_WEAK __attribute__ ((weak,unused))
 
 #endif // S3FS_COMMON_H_