Skip to content

Commit

Permalink
list: Decode key/prefix and next marker in Core layer (#1169)
Browse files Browse the repository at this point in the history 
Some modules depending on minio-go, such as Minio Gateway S3, depends
on minio-go Core to perform listing. However a regression is found
because decoding keys and common prefixes, along with marker for list v1
is not happening in Core module but in higher level.

This PR moves decoding to Core layer.

This PR also avoids encoding next continuation token since S3 spec
does not require it.
  • Loading branch information
@vadmeste @kannappanr
vadmeste authored and kannappanr committed Oct 1, 2019
1 parent 05a12f8 commit 23d7c54
Showing 1 changed file with 69 additions and 79 deletions.
148 changes: 69 additions & 79 deletions api-list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,6 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d

// If contents are available loop through and send over channel.
for _, object := range result.Contents {
object.Key, err = url.QueryUnescape(object.Key)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
select {
// Send object content.
case objectStatCh <- object:
Expand All @@ -147,17 +140,9 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
for _, obj := range result.CommonPrefixes {
objInfo := ObjectInfo{}
objInfo.Key, err = url.QueryUnescape(obj.Prefix)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
select {
// Send object prefixes.
case objectStatCh <- objInfo:
case objectStatCh <- ObjectInfo{Key: obj.Prefix}:
// If receives done from the caller, return here.
case <-doneCh:
return
Expand All @@ -166,13 +151,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d

// If continuation token present, save it for next request.
if result.NextContinuationToken != "" {
continuationToken, err = url.QueryUnescape(result.NextContinuationToken)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
continuationToken = result.NextContinuationToken
}

// Listing ends result is not truncated, return right here.
Expand Down Expand Up @@ -269,6 +248,20 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
return listBucketResult, errors.New("Truncated response should have continuation token set")
}

for _, obj := range listBucketResult.Contents {
obj.Key, err = url.QueryUnescape(obj.Key)
if err != nil {
return listBucketResult, err
}
}

for _, obj := range listBucketResult.CommonPrefixes {
obj.Prefix, err = url.QueryUnescape(obj.Prefix)
if err != nil {
return listBucketResult, err
}
}

// Success.
return listBucketResult, nil
}
Expand Down Expand Up @@ -338,13 +331,6 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don

// If contents are available loop through and send over channel.
for _, object := range result.Contents {
object.Key, err = url.QueryUnescape(object.Key)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
// Save the marker.
marker = object.Key
select {
Expand All @@ -359,18 +345,9 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
for _, obj := range result.CommonPrefixes {
object := ObjectInfo{}
object.Key, err = url.QueryUnescape(obj.Prefix)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
object.Size = 0
select {
// Send object prefixes.
case objectStatCh <- object:
case objectStatCh <- ObjectInfo{Key: obj.Prefix}:
// If receives done from the caller, return here.
case <-doneCh:
return
Expand All @@ -379,13 +356,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don

// If next marker present, save it for next request.
if result.NextMarker != "" {
marker, err = url.QueryUnescape(result.NextMarker)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
}
return
}
marker = result.NextMarker
}

// Listing ends result is not truncated, return right here.
Expand Down Expand Up @@ -461,6 +432,28 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
if err != nil {
return listBucketResult, err
}

for _, obj := range listBucketResult.Contents {
obj.Key, err = url.QueryUnescape(obj.Key)
if err != nil {
return listBucketResult, err
}
}

for _, obj := range listBucketResult.CommonPrefixes {
obj.Prefix, err = url.QueryUnescape(obj.Prefix)
if err != nil {
return listBucketResult, err
}
}

if listBucketResult.NextMarker != "" {
listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker)
if err != nil {
return listBucketResult, err
}
}

return listBucketResult, nil
}

Expand Down Expand Up @@ -531,21 +524,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
}
return
}
// Save objectMarker and uploadIDMarker for next request.
objectMarker, err = url.QueryUnescape(result.NextKeyMarker)
if err != nil {
objectMultipartStatCh <- ObjectMultipartInfo{
Err: err,
}
return
}
uploadIDMarker, err = url.QueryUnescape(result.NextUploadIDMarker)
if err != nil {
objectMultipartStatCh <- ObjectMultipartInfo{
Err: err,
}
return
}
objectMarker = result.NextKeyMarker
uploadIDMarker = result.NextUploadIDMarker

// Send all multipart uploads.
for _, obj := range result.Uploads {
// Calculate total size of the uploaded parts if 'aggregateSize' is enabled.
Expand All @@ -559,13 +540,6 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
continue
}
}
obj.Key, err = url.QueryUnescape(obj.Key)
if err != nil {
objectMultipartStatCh <- ObjectMultipartInfo{
Err: err,
}
return
}
select {
// Send individual uploads here.
case objectMultipartStatCh <- obj:
Expand All @@ -577,18 +551,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
for _, obj := range result.CommonPrefixes {
object := ObjectMultipartInfo{}
object.Key, err = url.QueryUnescape(obj.Prefix)
if err != nil {
objectMultipartStatCh <- ObjectMultipartInfo{
Err: err,
}
return
}
object.Size = 0
select {
// Send delimited prefixes here.
case objectMultipartStatCh <- object:
case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}:
// If done channel return here.
case <-doneCh:
return
Expand Down Expand Up @@ -666,6 +631,31 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
if err != nil {
return listMultipartUploadsResult, err
}

listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker)
if err != nil {
return listMultipartUploadsResult, err
}

listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker)
if err != nil {
return listMultipartUploadsResult, err
}

for _, obj := range listMultipartUploadsResult.Uploads {
obj.Key, err = url.QueryUnescape(obj.Key)
if err != nil {
return listMultipartUploadsResult, err
}
}

for _, obj := range listMultipartUploadsResult.CommonPrefixes {
obj.Prefix, err = url.QueryUnescape(obj.Prefix)
if err != nil {
return listMultipartUploadsResult, err
}
}

return listMultipartUploadsResult, nil
}

Expand Down

0 comments on commit 23d7c54

Please sign in to comment.