Fix listObjects hang when listing special characters (#1165)

This PR 1e01719 introduced
a regression in handling special character file listing, this
causes `mc ls` like commands to hang.

Fixes minio/mc#2903

api-list.go

@@ -128,6 +128,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 
 			// If contents are available loop through and send over channel.
 			for _, object := range result.Contents {
+				object.Key, err = url.QueryUnescape(object.Key)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 				select {
 				// Send object content.
 				case objectStatCh <- object:
@@ -140,12 +147,17 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 			// Send all common prefixes if any.
 			// NOTE: prefixes are only present if the request is delimited.
 			for _, obj := range result.CommonPrefixes {
+				objInfo := ObjectInfo{}
+				objInfo.Key, err = url.QueryUnescape(obj.Prefix)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 				select {
 				// Send object prefixes.
-				case objectStatCh <- ObjectInfo{
-					Key:  obj.Prefix,
-					Size: 0,
-				}:
+				case objectStatCh <- objInfo:
 				// If receives done from the caller, return here.
 				case <-doneCh:
 					return
@@ -154,7 +166,13 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 
 			// If continuation token present, save it for next request.
 			if result.NextContinuationToken != "" {
-				continuationToken = result.NextContinuationToken
+				continuationToken, err = url.QueryUnescape(result.NextContinuationToken)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 			}
 
 			// Listing ends result is not truncated, return right here.
@@ -320,6 +338,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
 
 			// If contents are available loop through and send over channel.
 			for _, object := range result.Contents {
+				object.Key, err = url.QueryUnescape(object.Key)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 				// Save the marker.
 				marker = object.Key
 				select {
@@ -335,7 +360,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
 			// NOTE: prefixes are only present if the request is delimited.
 			for _, obj := range result.CommonPrefixes {
 				object := ObjectInfo{}
-				object.Key = obj.Prefix
+				object.Key, err = url.QueryUnescape(obj.Prefix)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 				object.Size = 0
 				select {
 				// Send object prefixes.
@@ -348,7 +379,13 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
 
 			// If next marker present, save it for next request.
 			if result.NextMarker != "" {
-				marker = result.NextMarker
+				marker, err = url.QueryUnescape(result.NextMarker)
+				if err != nil {
+					objectStatCh <- ObjectInfo{
+						Err: err,
+					}
+					return
+				}
 			}
 
 			// Listing ends result is not truncated, return right here.
@@ -495,8 +532,20 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
 				return
 			}
 			// Save objectMarker and uploadIDMarker for next request.
-			objectMarker = result.NextKeyMarker
-			uploadIDMarker = result.NextUploadIDMarker
+			objectMarker, err = url.QueryUnescape(result.NextKeyMarker)
+			if err != nil {
+				objectMultipartStatCh <- ObjectMultipartInfo{
+					Err: err,
+				}
+				return
+			}
+			uploadIDMarker, err = url.QueryUnescape(result.NextUploadIDMarker)
+			if err != nil {
+				objectMultipartStatCh <- ObjectMultipartInfo{
+					Err: err,
+				}
+				return
+			}
 			// Send all multipart uploads.
 			for _, obj := range result.Uploads {
 				// Calculate total size of the uploaded parts if 'aggregateSize' is enabled.
@@ -510,6 +559,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
 						continue
 					}
 				}
+				obj.Key, err = url.QueryUnescape(obj.Key)
+				if err != nil {
+					objectMultipartStatCh <- ObjectMultipartInfo{
+						Err: err,
+					}
+					return
+				}
 				select {
 				// Send individual uploads here.
 				case objectMultipartStatCh <- obj:
@@ -522,7 +578,13 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
 			// NOTE: prefixes are only present if the request is delimited.
 			for _, obj := range result.CommonPrefixes {
 				object := ObjectMultipartInfo{}
-				object.Key = obj.Prefix
+				object.Key, err = url.QueryUnescape(obj.Prefix)
+				if err != nil {
+					objectMultipartStatCh <- ObjectMultipartInfo{
+						Err: err,
+					}
+					return
+				}
 				object.Size = 0
 				select {
 				// Send delimited prefixes here.
@@ -573,6 +635,9 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
 	// Set delimiter, delimiter value to be set to empty is okay.
 	urlValues.Set("delimiter", delimiter)
 
+	// Always set encoding-type
+	urlValues.Set("encoding-type", "url")
+
 	// maxUploads should be 1000 or less.
 	if maxUploads == 0 || maxUploads > 1000 {
 		maxUploads = 1000

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/minio/sha256-simd v0.1.0
+	github.com/minio/sha256-simd v0.1.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sirupsen/logrus v1.4.2 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect

go.sum

@@ -6,8 +6,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM=
-github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=