From 251c1ef6da4c79e9742c0391006ea65d26f73e2a Mon Sep 17 00:00:00 2001 From: Poorna Krishnamoorthy Date: Thu, 19 Nov 2020 11:50:22 -0800 Subject: [PATCH] Add support for replication of object tags, retention metadata (#10880) --- cmd/bucket-object-lock.go | 27 +++++---- cmd/bucket-replication.go | 111 +++++++++++++++++++++++++++++++---- cmd/erasure-object.go | 9 ++- cmd/erasure-server-sets.go | 1 + cmd/erasure-sets.go | 3 +- cmd/gateway/s3/gateway-s3.go | 2 +- cmd/object-handlers.go | 52 ++++++++++++++-- go.mod | 8 +-- go.sum | 12 ++++ 9 files changed, 187 insertions(+), 38 deletions(-) diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index 0eb83b0ad..c010e7232 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -21,10 +21,12 @@ import ( "math" "net/http" + xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" objectlock "github.com/minio/minio/pkg/bucket/object/lock" "github.com/minio/minio/pkg/bucket/policy" + "github.com/minio/minio/pkg/bucket/replication" ) // BucketObjectLockSys - map of bucket and retention configuration. @@ -245,13 +247,13 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, // For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered. // For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set // Both legal hold and retention can be applied independently on an object -func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) { +func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) { var mode objectlock.RetMode var retainDate objectlock.RetentionDate var legalHold objectlock.ObjectLegalHold - retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header) - legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header) + retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header) + legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header) retentionCfg, err := globalBucketObjectLockSys.Get(bucket) if err != nil { @@ -267,25 +269,24 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj return mode, retainDate, legalHold, ErrNone } - opts, err := getOpts(ctx, r, bucket, object) + opts, err := getOpts(ctx, rq, bucket, object) if err != nil { return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) } - if opts.VersionID != "" { + replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() + + if opts.VersionID != "" && !replica { if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil { r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined) - t, err := objectlock.UTCNowNTP() if err != nil { logger.LogIf(ctx, err) return mode, retainDate, legalHold, ErrObjectLocked } - if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) { return mode, retainDate, legalHold, ErrObjectLocked } - mode = r.Mode retainDate = r.RetainUntilDate legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined) @@ -298,17 +299,17 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj if legalHoldRequested { var lerr error - if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(r.Header); lerr != nil { + if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(rq.Header); lerr != nil { return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) } } if retentionRequested { - legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(r.Header) + legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(rq.Header) if err != nil { return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) } - rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(r.Header) + rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header) if err != nil { return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) } @@ -317,7 +318,9 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj } return rMode, rDate, legalHold, ErrNone } - + if replica { // replica inherits retention metadata only from source + return "", objectlock.RetentionDate{}, legalHold, ErrNone + } if !retentionRequested && retentionCfg.Validity > 0 { if retentionPermErr != ErrNone { return mode, retainDate, legalHold, retentionPermErr diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index add42e4f5..68d33a41d 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -24,6 +24,7 @@ import ( "strings" "time" + minio "github.com/minio/minio-go/v7" miniogo "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" @@ -249,6 +250,45 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA } } +func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string { + meta := make(map[string]string, len(oi.UserDefined)) + for k, v := range oi.UserDefined { + if k == xhttp.AmzBucketReplicationStatus { + continue + } + if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { + continue + } + meta[k] = v + } + if oi.ContentEncoding != "" { + meta[xhttp.ContentEncoding] = oi.ContentEncoding + } + if oi.ContentType != "" { + meta[xhttp.ContentType] = oi.ContentType + } + tag, err := tags.ParseObjectTags(oi.UserTags) + if err != nil { + return nil + } + if tag != nil { + meta[xhttp.AmzObjectTagging] = tag.String() + meta[xhttp.AmzTagDirective] = "REPLACE" + } + sc := dest.StorageClass + if sc == "" { + sc = oi.StorageClass + } + meta[xhttp.AmzStorageClass] = sc + if oi.UserTags != "" { + meta[xhttp.AmzObjectTagging] = oi.UserTags + } + meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339) + meta[xhttp.MinIOSourceETag] = oi.ETag + meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String() + return meta +} + func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) { meta := make(map[string]string) for k, v := range objInfo.UserDefined { @@ -302,6 +342,53 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn return } +type replicationAction string + +const ( + replicateMetadata replicationAction = "metadata" + replicateNone replicationAction = "none" + replicateAll replicationAction = "all" +) + +// returns replicationAction by comparing metadata between source and target +func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction { + // needs full replication + if oi1.ETag != oi2.ETag || + oi1.VersionID != oi2.VersionID || + oi1.Size != oi2.Size || + oi1.DeleteMarker != oi2.IsDeleteMarker { + return replicateAll + } + + if !oi1.ModTime.Equal(oi2.LastModified) || + oi1.ContentType != oi2.ContentType || + oi1.StorageClass != oi2.StorageClass { + return replicateMetadata + } + if oi1.ContentEncoding != "" { + enc, ok := oi2.UserMetadata[xhttp.ContentEncoding] + if !ok || enc != oi1.ContentEncoding { + return replicateMetadata + } + } + for k, v := range oi2.UserMetadata { + oi2.Metadata[k] = []string{v} + } + if len(oi2.Metadata) != len(oi1.UserDefined) { + return replicateMetadata + } + for k1, v1 := range oi1.UserDefined { + if v2, ok := oi2.Metadata[k1]; !ok || v1 != strings.Join(v2, "") { + return replicateMetadata + } + } + t, _ := tags.MapToObjectTags(oi2.UserTags) + if t.String() != oi1.UserTags { + return replicateMetadata + } + return replicateNone +} + // replicateObject replicates the specified version of the object to destination bucket // The source object is then updated to reflect the replication status. func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) { @@ -338,16 +425,11 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa return } - // if heal encounters a pending replication status, either replication - // has failed due to server shutdown or crawler and PutObject replication are in contention. - healPending := objInfo.ReplicationStatus == replication.Pending - - // In the rare event that replication is in pending state either due to - // server shut down/crash before replication completed or healing and PutObject - // race - do an additional stat to see if the version ID exists - if healPending { - _, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID}) - if err == nil { + rtype := replicateAll + oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID}) + if err == nil { + rtype = getReplicationAction(objInfo, oi) + if rtype == replicateNone { gr.Close() // object with same VersionID already exists, replication kicked off by // PutObject might have completed. @@ -375,8 +457,13 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa headerSize += len(k) + len(v) } r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit) - - _, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts) + if rtype == replicateAll { + _, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts) + } else { + // replicate metadata for object tagging/copy with metadata replacement + dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}} + _, err = tgt.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts) + } r.Close() if err != nil { replicationStatus = replication.Failed diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 05872d755..4b1690449 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -25,6 +25,7 @@ import ( "path" "strings" "sync" + "time" "github.com/minio/minio-go/v7/pkg/tags" xhttp "github.com/minio/minio/cmd/http" @@ -93,9 +94,12 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d } modTime = UTCNow() } - fi.VersionID = versionID // set any new versionID we might have created fi.ModTime = modTime // set modTime for the new versionID + if !dstOpts.MTime.IsZero() { + modTime = dstOpts.MTime + fi.ModTime = dstOpts.MTime + } srcInfo.UserDefined["etag"] = srcInfo.ETag @@ -1089,6 +1093,9 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin if tags != "" { fi.Metadata[xhttp.AmzObjectTagging] = tags } + for k, v := range opts.UserDefined { + fi.Metadata[k] = v + } metaArr[i].Metadata = fi.Metadata } diff --git a/cmd/erasure-server-sets.go b/cmd/erasure-server-sets.go index bd7bb0eb5..6da6b4968 100644 --- a/cmd/erasure-server-sets.go +++ b/cmd/erasure-server-sets.go @@ -632,6 +632,7 @@ func (z *erasureServerSets) CopyObject(ctx context.Context, srcBucket, srcObject UserDefined: srcInfo.UserDefined, Versioned: dstOpts.Versioned, VersionID: dstOpts.VersionID, + MTime: dstOpts.MTime, } return z.serverSets[zoneIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts) diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index dbc1d7fab..b25bc58af 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -774,10 +774,8 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB dstSet := s.getHashedSet(dstObject) cpSrcDstSame := srcSet == dstSet - // Check if this request is only metadata update. if cpSrcDstSame && srcInfo.metadataOnly { - // Version ID is set for the destination and source == destination version ID. // perform an in-place update. if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID { @@ -803,6 +801,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB UserDefined: srcInfo.UserDefined, Versioned: dstOpts.Versioned, VersionID: dstOpts.VersionID, + MTime: dstOpts.MTime, } return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts) diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 95ca0f1bb..1c3e2f1c2 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -510,7 +510,7 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject srcInfo.UserDefined[k] = v[0] } - if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil { + if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined, miniogo.PutObjectOptions{}); err != nil { return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject) } return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 9d0f7a091..c5c944bb3 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -902,7 +902,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) getObjectNInfo := objectAPI.GetObjectNInfo @@ -1164,7 +1163,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true) retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectRetentionAction) holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectLegalHoldAction) - getObjectInfo := objectAPI.GetObjectInfo if api.CacheAPI() != nil { getObjectInfo = api.CacheAPI().GetObjectInfo @@ -1183,10 +1181,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r)) return } + if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" { + srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs + } if mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String()) { srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String() } - // Store the preserved compression metadata. for k, v := range compressMetadata { srcInfo.UserDefined[k] = v @@ -1261,7 +1261,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } } - objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime) encodedSuccessResponse := encodeResponse(response) @@ -2853,16 +2852,24 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r if objInfo.UserTags != "" { objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags } + replicate := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "") + if replicate { + objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String() + } + objInfo.metadataOnly = true if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{ VersionID: opts.VersionID, }, ObjectOptions{ VersionID: opts.VersionID, + MTime: opts.MTime, }); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - + if replicate { + globalReplicationState.queueReplicaTask(objInfo) + } writeSuccessResponseHeadersOnly(w) // Notify object event. sendEvent(eventArgs{ @@ -3018,15 +3025,23 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r if objInfo.UserTags != "" { objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags } + replicate := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "") + if replicate { + objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String() + } objInfo.metadataOnly = true // Perform only metadata updates. if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{ VersionID: opts.VersionID, }, ObjectOptions{ VersionID: opts.VersionID, + MTime: opts.MTime, }); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } + if replicate { + globalReplicationState.queueReplicaTask(objInfo) + } writeSuccessNoContent(w) // Notify object event. @@ -3192,6 +3207,12 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h return } + replicate := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: tags.String()}, "") + if replicate { + opts.UserDefined = make(map[string]string) + opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String() + } + // Put object tags err = objAPI.PutObjectTags(ctx, bucket, object, tags.String(), opts) if err != nil { @@ -3199,6 +3220,12 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h return } + if replicate { + if objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil { + globalReplicationState.queueReplicaTask(objInfo) + } + } + if opts.VersionID != "" { w.Header()[xhttp.AmzVersionID] = []string{opts.VersionID} } @@ -3240,7 +3267,16 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - + oi, err := objAPI.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + replicate := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: oi.UserTags}, "") + if replicate { + opts.UserDefined = make(map[string]string) + opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String() + } // Delete object tags if err = objAPI.DeleteObjectTags(ctx, bucket, object, opts); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) @@ -3251,6 +3287,10 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r w.Header()[xhttp.AmzVersionID] = []string{opts.VersionID} } + if replicate { + globalReplicationState.queueReplicaTask(oi) + } + writeSuccessNoContent(w) } diff --git a/go.mod b/go.mod index d25100f80..fca374ccf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/bcicen/jstream v1.0.1 github.com/beevik/ntp v0.3.0 github.com/cespare/xxhash/v2 v2.1.1 - github.com/cheggaaa/pb v1.0.28 + github.com/cheggaaa/pb v1.0.29 github.com/colinmarc/hdfs/v2 v2.1.1 github.com/coredns/coredns v1.4.0 github.com/dchest/siphash v1.2.1 @@ -24,7 +24,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/eclipse/paho.mqtt.golang v1.2.0 github.com/elazarl/go-bindata-assetfs v1.0.0 - github.com/fatih/color v1.7.0 + github.com/fatih/color v1.9.0 github.com/fatih/structs v1.1.0 github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-sql-driver/mysql v1.5.0 @@ -44,11 +44,11 @@ require ( github.com/lib/pq v1.8.0 github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-ieproxy v0.0.1 // indirect - github.com/mattn/go-isatty v0.0.8 + github.com/mattn/go-isatty v0.0.11 github.com/miekg/dns v1.1.8 github.com/minio/cli v1.22.0 github.com/minio/highwayhash v1.0.0 - github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac + github.com/minio/minio-go/v7 v7.0.6-0.20201118225257-f6869a5e2a6a github.com/minio/selfupdate v0.3.1 github.com/minio/sha256-simd v0.1.1 github.com/minio/simdjson-go v0.1.5 diff --git a/go.sum b/go.sum index 727e2114a..d2e4ba82e 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -106,6 +108,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -298,8 +302,12 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI= @@ -312,6 +320,8 @@ github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac h1:0meQIZTQR/JkAxfygReKcb15QINBKpFd4LII2PT5jSY= github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= +github.com/minio/minio-go/v7 v7.0.6-0.20201118225257-f6869a5e2a6a h1:sPLqZWtSR1G57sQq/b5RDEADZ5t++JuIwBxu7NvicMY= +github.com/minio/minio-go/v7 v7.0.6-0.20201118225257-f6869a5e2a6a/go.mod h1:HcIuq+11d/3MfavIPZiswSzfQ1VJ2Lwxp/XLtW46IWQ= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -417,6 +427,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -554,6 +565,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=