From 89d5404ffa4a81b3e719a335352edf902b0efa95 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 23 Sep 2019 13:05:38 -0700 Subject: [PATCH] Fix listObjects hang when listing special characters (#1165) This PR 1e017194bd0a9f77db34525b4592e73f29ced561 introduced a regression in handling special character file listing, this causes `mc ls` like commands to hang. Fixes https://github.com/minio/mc/issues/2903 --- api-list.go | 85 ++++++++++++++++++++++++++++++++++++++++++++++------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/api-list.go b/api-list.go index c7368fcbf4..0c6a4681fc 100644 --- a/api-list.go +++ b/api-list.go @@ -128,6 +128,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If contents are available loop through and send over channel. for _, object := range result.Contents { + object.Key, err = url.QueryUnescape(object.Key) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } select { // Send object content. case objectStatCh <- object: @@ -140,12 +147,17 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { + objInfo := ObjectInfo{} + objInfo.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } select { // Send object prefixes. - case objectStatCh <- ObjectInfo{ - Key: obj.Prefix, - Size: 0, - }: + case objectStatCh <- objInfo: // If receives done from the caller, return here. case <-doneCh: return @@ -154,7 +166,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // If continuation token present, save it for next request. if result.NextContinuationToken != "" { - continuationToken = result.NextContinuationToken + continuationToken, err = url.QueryUnescape(result.NextContinuationToken) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } } // Listing ends result is not truncated, return right here. @@ -320,6 +338,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If contents are available loop through and send over channel. for _, object := range result.Contents { + object.Key, err = url.QueryUnescape(object.Key) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } // Save the marker. marker = object.Key select { @@ -335,7 +360,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { object := ObjectInfo{} - object.Key = obj.Prefix + object.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } object.Size = 0 select { // Send object prefixes. @@ -348,7 +379,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // If next marker present, save it for next request. if result.NextMarker != "" { - marker = result.NextMarker + marker, err = url.QueryUnescape(result.NextMarker) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } } // Listing ends result is not truncated, return right here. @@ -495,8 +532,20 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive return } // Save objectMarker and uploadIDMarker for next request. - objectMarker = result.NextKeyMarker - uploadIDMarker = result.NextUploadIDMarker + objectMarker, err = url.QueryUnescape(result.NextKeyMarker) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } + uploadIDMarker, err = url.QueryUnescape(result.NextUploadIDMarker) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. @@ -510,6 +559,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive continue } } + obj.Key, err = url.QueryUnescape(obj.Key) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } select { // Send individual uploads here. case objectMultipartStatCh <- obj: @@ -522,7 +578,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { object := ObjectMultipartInfo{} - object.Key = obj.Prefix + object.Key, err = url.QueryUnescape(obj.Prefix) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } object.Size = 0 select { // Send delimited prefixes here. @@ -573,6 +635,9 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) + // Always set encoding-type + urlValues.Set("encoding-type", "url") + // maxUploads should be 1000 or less. if maxUploads == 0 || maxUploads > 1000 { maxUploads = 1000 diff --git a/go.mod b/go.mod index d728ac7b85..2dc189d1aa 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/dustin/go-humanize v1.0.0 // indirect - github.com/minio/sha256-simd v0.1.0 + github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.4.2 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect diff --git a/go.sum b/go.sum index a8ad8bde43..b3cacbda4d 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= -github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=