Merge pull request #173 from aliyun/release_1.91.5

release 1.91.5

ChangeLog

@@ -1,5 +1,17 @@
 ChangeLog for OSSFS
 ------------------
+## v1.91.5 (25/12/2024)
+ - Fixed some bugs that users get unexpected results on mounting with specific policies for the bucket/prefix set.
+ - Fixed a bug that ossfs take a file as existing if HeadObj returns 403.
+ - Change the default value of complement_stat to true.
+ - Support free_space_ratio.
+
+## v1.91.4 (24/09/2024)
+ - Add a new option sigv4 to support OSS V4 signature. 
+ - Fixed a bug that concurrent writing may cause deadlock when disk space is insufficient.
+ - Support authentication via dynamic lib.
+ - Fixed a bug that AK and SK are stored in wrong environment variables.
+
 ## v1.91.3 (04/06/2024)
  - Fixed a bug that IO fails when running out of disk space.
  - Fixed a bug that default_acl option does not working.

configure.ac

@@ -20,7 +20,7 @@
 dnl Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.59)
-AC_INIT(ossfs, 1.91.4)
+AC_INIT(ossfs, 1.91.5)
 AC_CONFIG_HEADER([config.h])
 
 AC_CANONICAL_SYSTEM

doc/man/ossfs.1.in

@@ -187,6 +187,12 @@ sets MB to ensure disk free space. This option means the threshold of free space
 ossfs makes file for downloading, uploading and caching files.
 If the disk free space is smaller than this value, ossfs do not use disk space as possible in exchange for the performance.
 .TP
+\fB\-o\fR free_space_ratio (default="0")
+sets min free space ratio of the disk. The value of this option can be between 0 and 100. It will control
+the size of the cache according to this ratio to ensure that the idle ratio of the disk is greater than this value.
+For example, when the disk space is 50GB, the 10% value will
+ensure that the disk will reserve at least 50GB * 10% = 5GB of remaining space.
+.TP
 \fB\-o\fR multipart_threshold (default="25")
 threshold, in MB, to use multipart upload instead of
 single-part. Must be at least 5 MB.
@@ -291,9 +297,9 @@ https://curl.haxx.se/docs/ssl-ciphers.html
 The instance name of the current ossfs mountpoint.
 This name will be added to logging messages and user agent headers sent by ossfs.
 .TP
-\fB\-o\fR complement_stat (complement lack of file/directory mode)
+\fB\-o\fR complement_stat (complement lack of file/directory mode, enabled by default)
 ossfs complements lack of information about file/directory mode if a file or a directory object does not have x-oss-meta-mode header.
-As default, ossfs does not complements stat information for a object, then the object will not be able to be allowed to list/modify.
+If disabled, ossfs does not complements stat information for a object, then the object will not be able to be allowed to list/modify.
 .TP
 \fB\-o\fR notsup_compat_dir (disable support of alternative directory names)
 .RS

scripts/ossfs-coverage-centos7.sh

@@ -34,6 +34,10 @@ rm -rf ${OSSFS_SOURCE_DIR}/coverage_html && mkdir ${OSSFS_SOURCE_DIR}/coverage_h
 
 DBGLEVEL=debug ALL_TESTS=1 OSSFS_CREDENTIALS_FILE=/root/.passwd-ossfs TEST_BUCKET_1=${BUCKET}  S3PROXY_BINARY="" OSS_URL=${URL} ./small-integration-test.sh
 
+${OSSFS_SOURCE_DIR}/src/test_page_list 
+${OSSFS_SOURCE_DIR}/src/test_curl_util
+${OSSFS_SOURCE_DIR}/src/test_string_util
+
 gcovr -r ${OSSFS_SOURCE_DIR}/src --html-details -o ${OSSFS_SOURCE_DIR}/coverage_html/coverage.html
 
 

src/fdcache.cpp

@@ -272,6 +272,11 @@ bool FdManager::InitFakeUsedDiskSize(off_t fake_freesize)
     return true;
 }
 
+off_t FdManager::GetTotalDiskSpaceByRatio(int ratio)
+{
+    return FdManager::GetTotalDiskSpace(nullptr) * ratio / 100;
+}
+
 off_t FdManager::GetTotalDiskSpace(const char* path)
 {
     struct statvfs vfsbuf;
@@ -328,6 +333,18 @@ bool FdManager::IsSafeDiskSpace(const char* path, off_t size)
     return size + FdManager::GetEnsureFreeDiskSpace() <= fsize;
 }
 
+bool FdManager::IsSafeDiskSpaceWithLog(const char* path, off_t size)
+{
+    off_t fsize = FdManager::GetFreeDiskSpace(path);
+    off_t needsize = size + FdManager::GetEnsureFreeDiskSpace();
+    if(needsize <= fsize){
+        return true;
+    } else {
+        S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by ossfs. Requires %.3f MB, already has %.3f MB.", static_cast<double>(needsize) / 1024 / 1024, static_cast<double>(fsize) / 1024 / 1024);
+        return false;
+    }
+}
+
 bool FdManager::HaveLseekHole()
 {
     if(FdManager::checked_lseek){

src/fdcache.h

@@ -84,12 +84,14 @@ class FdManager
       static off_t SetEnsureFreeDiskSpace(off_t size);
       static bool InitFakeUsedDiskSize(off_t fake_freesize);
       static bool IsSafeDiskSpace(const char* path, off_t size);
+      static bool IsSafeDiskSpaceWithLog(const char* path, off_t size);
       static void FreeReservedDiskSpace(off_t size);
       static bool ReserveDiskSpace(off_t size);
       static bool HaveLseekHole();
       static bool SetTmpDir(const char* dir);
       static bool CheckTmpDirExist();
       static FILE* MakeTempFile();
+      static off_t GetTotalDiskSpaceByRatio(int ratio);
 
       // Return FdEntity associated with path, returning NULL on error.  This operation increments the reference count; callers must decrement via Close after use.
       FdEntity* GetFdEntity(const char* path, int& existfd, bool newfd = true, bool lock_already_held = false);

src/s3fs.cpp

@@ -131,7 +131,6 @@ static int rename_object(const char* from, const char* to, bool update_ctime);
 static int rename_object_nocopy(const char* from, const char* to, bool update_ctime);
 static int clone_directory_object(const char* from, const char* to, bool update_ctime);
 static int rename_directory(const char* from, const char* to);
-static int remote_mountpath_exists(const char* path);
 static void free_xattrs(xattrs_t& xattrs);
 static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, PXATTRVAL& pval);
 static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
@@ -396,26 +395,7 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
     s3fscurl.DestroyCurlHandle();
 
     // if not found target path object, do over checking
-    if(-EPERM == result){
-        // [NOTE]
-        // In case of a permission error, it exists in directory
-        // file list but inaccessible. So there is a problem that
-        // it will send a HEAD request every time, because it is
-        // not registered in the Stats cache.
-        // Therefore, even if the file has a permission error, it
-        // should be registered in the Stats cache. However, if
-        // the response without modifying is registered in the
-        // cache, the file permission will be 0644(umask dependent)
-        // because the meta header does not exist.
-        // Thus, set the mode of 0000 here in the meta header so
-        // that ossfs can print a permission error when the file
-        // is actually accessed.
-        // It is better not to set meta header other than mode,
-        // so do not do it.
-        //
-        (*pheader)["x-oss-meta-mode"] = str(0);
-
-    }else if(0 != result){
+   if(0 != result){
         if(overcheck){
             // when support_compat_dir is disabled, strpath maybe have "_$folder$".
             if('/' != *strpath.rbegin() && std::string::npos == strpath.find("_$folder$", 0)){
@@ -467,7 +447,7 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
     // If the file is listed but not allowed access, put it in
     // the positive cache instead of the negative cache.
     // 
-    if(0 != result && -EPERM != result){
+    if(0 != result){
         // finally, "path" object did not find. Add no object cache.
         strpath = path;  // reset original
         StatCache::getStatCacheData()->AddNoObjectCache(strpath);
@@ -3031,24 +3011,6 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
     return 0;
 }
 
-static int remote_mountpath_exists(const char* path)
-{
-    struct stat stbuf;
-    int result;
-
-    S3FS_PRN_INFO1("[path=%s]", path);
-
-    // getattr will prefix the path with the remote mountpoint
-    if(0 != (result = get_object_attribute("/", &stbuf, NULL))){
-        return result;
-    }
-    if(!S_ISDIR(stbuf.st_mode)){
-        return -ENOTDIR;
-    }
-    return 0;
-}
-
-
 static void free_xattrs(xattrs_t& xattrs)
 {
     for(xattrs_t::iterator iter = xattrs.begin(); iter != xattrs.end(); ++iter){
@@ -3777,7 +3739,7 @@ static int s3fs_check_service()
 
     S3fsCurl s3fscurl;
     int      res;
-    if(0 > (res = s3fscurl.CheckBucket("/"))){
+    if(0 > (res = s3fscurl.CheckBucket(get_realpath("/").c_str()))){
         // get response code
         long responseCode = s3fscurl.GetLastResponseCode();
 
@@ -3828,7 +3790,9 @@ static int s3fs_check_service()
                 } else {
                     s3host = "http://" + expecthost;
                 }
-                //extract region from host for sigv4
+                // extract region from host for sigv4
+                // The region of the government cloud/financial cloud may not be derived from the domain name
+                // https://help.aliyun.com/zh/oss/user-guide/regions-and-endpoints?spm=a2c4g.11186623.0.0.13ad12c1N7PoaV
                 if(!strncasecmp(expecthost.c_str(), "oss-", 4)){
                     std::size_t found;
                     if ((found = expecthost.find_first_of(".")) != std::string::npos) {
@@ -3838,18 +3802,11 @@ static int s3fs_check_service()
                 }
                 // retry to check with new host
                 s3fscurl.DestroyCurlHandle();
-                res          = s3fscurl.CheckBucket("/");
+                res          = s3fscurl.CheckBucket(get_realpath("/").c_str());
                 responseCode = s3fscurl.GetLastResponseCode();
             }
         }
 
-        // retry to check with mount prefix
-        if(300 <= responseCode && responseCode < 500 && !mount_prefix.empty()){
-            s3fscurl.DestroyCurlHandle();
-            res          = s3fscurl.CheckBucket(get_realpath("/").c_str());
-            responseCode = s3fscurl.GetLastResponseCode();
-        }
-
         // try signature v2
         /*
         if(0 > res && (responseCode == 400 || responseCode == 403) && S3fsCurl::GetSignatureType() == V1_OR_V4){
@@ -3864,35 +3821,39 @@ static int s3fs_check_service()
         }
         */
         // check errors(after retrying)
+        // [NOTE]
+        // When mounting a bucket, an error code is returned and the mount fails. 
+        // However, when mounting a prefix, success should be returned if the prefix does not exist.
+        // 
         if(0 > res && responseCode != 200 && responseCode != 301){
             // parse error message if existed
             std::string errMessage;
             const std::string* body = s3fscurl.GetBodyData();
             check_error_message(body->c_str(), body->size(), errMessage);
-
+            bool is_failure = true;
             if(responseCode == 400){
                 S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Bad Request(host=%s, message=%s)", s3host.c_str(), errMessage.c_str());
             }else if(responseCode == 403){
                  S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Invalid Credentials(host=%s, message=%s)", s3host.c_str(), errMessage.c_str());
-            }else if(responseCode == 404){
-                S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Bucket or directory not found(host=%s, message=%s)", s3host.c_str(), errMessage.c_str());
+            }else if (responseCode == 404) {
+                std::string value;
+                if(simple_parse_xml(body->c_str(), body->size(), "Code", value)) {
+                    if(value == "NoSuchBucket") {
+                        S3FS_PRN_CRIT("Failed to check bucket : Bucket not found(host=%s, message=%s)", s3host.c_str(), errMessage.c_str());
+                    } else {
+                        is_failure = false;
+                    }
+                }
             }else{
                 S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Unable to connect(host=%s, message=%s)", s3host.c_str(), errMessage.c_str());
             }
-            return EXIT_FAILURE;
-        }
-    }
-    s3fscurl.DestroyCurlHandle();
-
-    // make sure remote mountpath exists and is a directory
-    if(!mount_prefix.empty()){
-        if(remote_mountpath_exists(mount_prefix.c_str()) != 0){
-            S3FS_PRN_CRIT("remote mountpath %s not found.", mount_prefix.c_str());
-            return EXIT_FAILURE;
+            if (is_failure) {
+                return EXIT_FAILURE;
+            }
         }
     }
     S3FS_MALLOCTRIM(0);
-
+    
     return EXIT_SUCCESS;
 }
 
@@ -4407,8 +4368,34 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             max_dirty_data = size;
             return 0;
         }
+        if(is_prefix(arg, "free_space_ratio=")){
+            int ratio = static_cast<int>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
+            if(FdManager::GetEnsureFreeDiskSpace()!=0){
+                S3FS_PRN_EXIT("option free_space_ratio conflicts with ensure_diskfree, please set only one of them.");
+                return -1;
+            }
+            if(ratio < 0 || ratio > 100){
+                S3FS_PRN_EXIT("option free_space_ratio must between 0 to 100, which is: %d", ratio);
+                return -1;
+            }
+            off_t dfsize = FdManager::GetTotalDiskSpaceByRatio(ratio);
+            S3FS_PRN_INFO("Free space ratio set to %d %%, ensure the available disk space is greater than %.3f MB", ratio, static_cast<double>(dfsize) / 1024 / 1024);
+            if(dfsize < S3fsCurl::GetMultipartSize()){
+                S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it.");
+                dfsize = S3fsCurl::GetMultipartSize();
+            }
+            FdManager::SetEnsureFreeDiskSpace(dfsize);
+            return 0;
+        }
         if(is_prefix(arg, "ensure_diskfree=")){
             off_t dfsize = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024;
+
+            if(FdManager::GetEnsureFreeDiskSpace()!=0){
+                S3FS_PRN_EXIT("option free_space_ratio conflicts with ensure_diskfree, please set only one of them.");
+                return -1;
+            }
+            S3FS_PRN_INFO("Set and ensure the available disk space is greater than %.3f MB.", static_cast<double>(dfsize) / 1024 / 1024);
+
             if(dfsize < S3fsCurl::GetMultipartSize()){
                 S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it.");
                 dfsize = S3fsCurl::GetMultipartSize();
@@ -5078,12 +5065,16 @@ int main(int argc, char* argv[])
 
     // check free disk space
     if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){
-        S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs.");
-        S3fsCurl::DestroyS3fsCurl();
-        s3fs_destroy_global_ssl();
-        destroy_parser_xml_lock();
-        delete ps3fscred;
-        exit(EXIT_FAILURE);
+        // clean cache dir and retry
+        S3FS_PRN_WARN("No enough disk space for ossfs, try to clean cache dir");
+        FdManager::get()->CleanupCacheDir();
+        if(!FdManager::IsSafeDiskSpaceWithLog(nullptr, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){
+            S3fsCurl::DestroyS3fsCurl();
+            s3fs_destroy_global_ssl();
+            destroy_parser_xml_lock();
+            delete ps3fscred;
+            exit(EXIT_FAILURE);
+        }
     }
 
     // check readdir_optimize 
@@ -5101,6 +5092,11 @@ int main(int argc, char* argv[])
         S3FS_PRN_INFO("Readdir optimize, flag(%d, %lld)", is_refresh_fakemeta, static_cast<long long int>(readdir_check_size));
     }
 
+    // try to check s3fs service
+    if (EXIT_SUCCESS != s3fs_check_service()) {
+        exit(EXIT_FAILURE);
+    }
+
     s3fs_oper.getattr     = s3fs_getattr;
     s3fs_oper.readlink    = s3fs_readlink;
     s3fs_oper.mknod       = s3fs_mknod;

src/s3fs_global.cpp

@@ -27,7 +27,7 @@
 bool foreground                   = false;
 bool nomultipart                  = false;
 bool pathrequeststyle             = false;
-bool complement_stat              = false;
+bool complement_stat              = true;
 bool noxmlns                      = true;
 bool direct_read                  = false;
 

src/s3fs_help.cpp

@@ -219,6 +219,15 @@ static const char help_string[] =
     "        space is smaller than this value, ossfs do not use disk space\n"
     "        as possible in exchange for the performance.\n"
     "\n"
+    "   free_space_ratio (default=\"0\")\n"
+    "      - sets min free space ratio of the disk.\n"
+    "      The value of this option can be between 0 and 100. It will control\n"
+    "      the size of the cache according to this ratio to ensure that the\n"
+    "      idle ratio of the disk is greater than this value.\n"
+    "      For example, when the disk space is 50GB, the default value will\n"
+    "      ensure that the disk will reserve at least 50GB * 10%% = 5GB of\n"
+    "      remaining space.\n"
+    "\n"
     "   multipart_threshold (default=\"25\")\n"
     "      - threshold, in MB, to use multipart upload instead of\n"
     "        single-part. Must be at least 5 MB.\n"
@@ -357,10 +366,10 @@ static const char help_string[] =
     "   instance_name - The instance name of the current ossfs mountpoint.\n"
     "        This name will be added to logging messages and user agent headers sent by ossfs.\n"
     "\n"
-    "   complement_stat (complement lack of file/directory mode)\n"
+    "   complement_stat (complement lack of file/directory mode, enabled by default)\n"
     "        ossfs complements lack of information about file/directory mode\n"
     "        if a file or a directory object does not have x-oss-meta-mode\n"
-    "        header. As default, ossfs does not complements stat information\n"
+    "        header. If disabled, ossfs does not complements stat information\n"
     "        for a object, then the object will not be able to be allowed to\n"
     "        list/modify.\n"
     "\n"