MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 06:05:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

azure: handle list objects properly (#4953)

Browse Source 
When removing `minio.sys.tmp` prefixed entries and objects/prefixes is
empty, populate till we get all valid entries.
This commit is contained in:
Bala FA 2017-09-29 12:08:23 -07:00 committed by Dee Koder
parent ce2d185211
commit 60cc6184d2
8 changed files with 62 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/gateway-azure-anonymous.go
View File

@ -271,7 +271,7 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
} }
// AnonListObjectsV2 - List objects in V2 mode, anonymously // AnonListObjectsV2 - List objects in V2 mode, anonymously
func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) { func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
params := storage.ListBlobsParameters{ params := storage.ListBlobsParameters{
Prefix: prefix, Prefix: prefix,
Marker: continuationToken, Marker: continuationToken,

119
cmd/gateway-azure.go
View File

@ -385,84 +385,73 @@ func (a *azureObjects) DeleteBucket(bucket string) error {
// ListObjects - lists all blobs on azure with in a container filtered by prefix // ListObjects - lists all blobs on azure with in a container filtered by prefix
// and marker, uses Azure equivalent ListBlobs. // and marker, uses Azure equivalent ListBlobs.
func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) { func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
var objects []ObjectInfo
var prefixes []string
container := a.client.GetContainerReference(bucket) container := a.client.GetContainerReference(bucket)
resp, err := container.ListBlobs(storage.ListBlobsParameters{ for len(objects) == 0 && len(prefixes) == 0 {
Prefix: prefix, resp, err := container.ListBlobs(storage.ListBlobsParameters{
Marker: marker, Prefix: prefix,
Delimiter: delimiter, Marker: marker,
MaxResults: uint(maxKeys), Delimiter: delimiter,
}) MaxResults: uint(maxKeys),
if err != nil {
return result, azureToObjectError(traceError(err), bucket, prefix)
}
result.IsTruncated = resp.NextMarker != ""
result.NextMarker = resp.NextMarker
for _, object := range resp.Blobs {
if strings.HasPrefix(object.Name, globalMinioSysTmp) {
continue
}
result.Objects = append(result.Objects, ObjectInfo{
Bucket: bucket,
Name: object.Name,
ModTime: time.Time(object.Properties.LastModified),
Size: object.Properties.ContentLength,
ETag: azureToS3ETag(object.Properties.Etag),
ContentType: object.Properties.ContentType,
ContentEncoding: object.Properties.ContentEncoding,
}) })
} if err != nil {
return result, azureToObjectError(traceError(err), bucket, prefix)
}
// Remove minio.sys.tmp prefix. for _, object := range resp.Blobs {
for i, prefix := range resp.BlobPrefixes { if strings.HasPrefix(object.Name, globalMinioSysTmp) {
if prefix == globalMinioSysTmp { continue
resp.BlobPrefixes = append(resp.BlobPrefixes[:i], resp.BlobPrefixes[i+1:]...) }
objects = append(objects, ObjectInfo{
Bucket: bucket,
Name: object.Name,
ModTime: time.Time(object.Properties.LastModified),
Size: object.Properties.ContentLength,
ETag: azureToS3ETag(object.Properties.Etag),
ContentType: object.Properties.ContentType,
ContentEncoding: object.Properties.ContentEncoding,
})
}
// Remove minio.sys.tmp prefix.
for _, prefix := range resp.BlobPrefixes {
if prefix != globalMinioSysTmp {
prefixes = append(prefixes, prefix)
}
}
marker = resp.NextMarker
if resp.NextMarker == "" {
break break
} }
} }
result.Prefixes = resp.BlobPrefixes
result.Objects = objects
result.Prefixes = prefixes
result.NextMarker = marker
result.IsTruncated = (marker != "")
return result, nil return result, nil
} }
// ListObjectsV2 - list all blobs in Azure bucket filtered by prefix // ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) { func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
container := a.client.GetContainerReference(bucket) marker := continuationToken
resp, err := container.ListBlobs(storage.ListBlobsParameters{ if startAfter != "" {
Prefix: prefix, marker = startAfter
Marker: continuationToken,
Delimiter: delimiter,
MaxResults: uint(maxKeys),
})
if err != nil {
return result, azureToObjectError(traceError(err), bucket, prefix)
}
// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
if resp.NextMarker != "" {
result.IsTruncated = true
result.NextContinuationToken = resp.NextMarker
}
for _, object := range resp.Blobs {
if strings.HasPrefix(object.Name, globalMinioSysTmp) {
continue
}
result.Objects = append(result.Objects, ObjectInfo{
Bucket: bucket,
Name: object.Name,
ModTime: time.Time(object.Properties.LastModified),
Size: object.Properties.ContentLength,
ETag: azureToS3ETag(object.Properties.Etag),
ContentType: object.Properties.ContentType,
ContentEncoding: object.Properties.ContentEncoding,
})
} }
// Remove minio.sys.tmp prefix. var resultV1 ListObjectsInfo
for i, prefix := range resp.BlobPrefixes { resultV1, err = a.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
if prefix == globalMinioSysTmp { if err != nil {
resp.BlobPrefixes = append(resp.BlobPrefixes[:i], resp.BlobPrefixes[i+1:]...) return result, err
break
}
} }
result.Prefixes = resp.BlobPrefixes
result.Objects = resultV1.Objects
result.Prefixes = resultV1.Prefixes
result.ContinuationToken = continuationToken
result.NextContinuationToken = resultV1.NextMarker
result.IsTruncated = (resultV1.NextMarker != "")
return result, nil return result, nil
} }

2
cmd/gateway-gcs-anonymous.go
View File

@ -112,7 +112,7 @@ func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string
} }
// AnonListObjectsV2 - List objects in V2 mode, anonymously // AnonListObjectsV2 - List objects in V2 mode, anonymously
func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) { func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
// Request V1 List Object to the backend // Request V1 List Object to the backend
result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys) result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys)
if err != nil { if err != nil {

3
cmd/gateway-gcs.go
View File

@ -562,8 +562,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
} }
// ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
delimiter string, maxKeys int) (ListObjectsV2Info, error) {
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
Delimiter: delimiter, Delimiter: delimiter,

2
cmd/gateway-handlers.go
View File

@ -810,7 +810,7 @@ func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *htt
// Inititate a list objects operation based on the input params. // Inititate a list objects operation based on the input params.
// On success would return back ListObjectsV2Info object to be // On success would return back ListObjectsV2Info object to be
// serialized as XML and sent as S3 compatible response body. // serialized as XML and sent as S3 compatible response body.
listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, fetchOwner, delimiter, maxKeys) listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
if err != nil { if err != nil {
errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter) errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter)
writeErrorResponse(w, toAPIErrorCode(err), r.URL) writeErrorResponse(w, toAPIErrorCode(err), r.URL)

4
cmd/gateway-router.go
View File

@ -36,8 +36,8 @@ type GatewayLayer interface {
GetBucketPolicies(string) (policy.BucketAccessPolicy, error) GetBucketPolicies(string) (policy.BucketAccessPolicy, error)
DeleteBucketPolicies(string) error DeleteBucketPolicies(string) error
AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
} }

2
cmd/gateway-s3-anonymous.go
View File

@ -95,7 +95,7 @@ func (l *s3Objects) AnonListObjects(bucket string, prefix string, marker string,
} }
// AnonListObjectsV2 - List objects in V2 mode, anonymously // AnonListObjectsV2 - List objects in V2 mode, anonymously
func (l *s3Objects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (loi ListObjectsV2Info, e error) { func (l *s3Objects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
result, err := l.anonClient.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) result, err := l.anonClient.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
if err != nil { if err != nil {
return loi, s3ToObjectError(traceError(err), bucket) return loi, s3ToObjectError(traceError(err), bucket)

2
cmd/gateway-s3.go
View File

@ -230,7 +230,7 @@ func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, del
} }
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (loi ListObjectsV2Info, e error) { func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
if err != nil { if err != nil {
return loi, s3ToObjectError(traceError(err), bucket) return loi, s3ToObjectError(traceError(err), bucket)