diff --git a/.github/workflows/go-cross.yml b/.github/workflows/go-cross.yml index ef3d39bad..7ca649d53 100644 --- a/.github/workflows/go-cross.yml +++ b/.github/workflows/go-cross.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] os: [ubuntu-latest] steps: - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 diff --git a/.github/workflows/go-healing.yml b/.github/workflows/go-healing.yml index 2e9984d65..f007a23da 100644 --- a/.github/workflows/go-healing.yml +++ b/.github/workflows/go-healing.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] os: [ubuntu-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml index 6aff20b82..568822024 100644 --- a/.github/workflows/go-lint.yml +++ b/.github/workflows/go-lint.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d37e2dc4b..8b9a75c7a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] os: [ubuntu-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/replication.yaml b/.github/workflows/replication.yaml index 79227d3f2..5356fa9ad 100644 --- a/.github/workflows/replication.yaml +++ b/.github/workflows/replication.yaml @@ -21,7 +21,7 @@ jobs: strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/upgrade-ci-cd.yaml b/.github/workflows/upgrade-ci-cd.yaml index eee542adf..6a110f127 100644 --- a/.github/workflows/upgrade-ci-cd.yaml +++ b/.github/workflows/upgrade-ci-cd.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.18.x] + go-version: [1.18.x, 1.19.x] os: [ubuntu-latest] steps: diff --git a/.golangci.yml b/.golangci.yml index 6511cde25..199eab226 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,8 +1,9 @@ linters-settings: golint: min-confidence: 0 + gofumpt: - lang-version: "1.17" + lang-version: "1.18" # Choose whether or not to use the extra rules that are disabled # by default @@ -20,11 +21,10 @@ linters: - govet - revive - ineffassign - - deadcode - gomodguard - gofmt - unconvert - - varcheck + - unused - gocritic - gofumpt - tenv diff --git a/Makefile b/Makefile index bc39290a3..0c5c3a37e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ help: ## print this help getdeps: ## fetch necessary dependencies @mkdir -p ${GOPATH}/bin - @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2 + @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin @echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e @echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest diff --git a/cmd/api-utils.go b/cmd/api-utils.go index fa277b9fc..c80a5cd82 100644 --- a/cmd/api-utils.go +++ b/cmd/api-utils.go @@ -35,8 +35,8 @@ func shouldEscape(c byte) bool { // s3URLEncode is based on Golang's url.QueryEscape() code, // while considering some S3 exceptions: -// - Avoid encoding '/' and '*' -// - Force encoding of '~' +// - Avoid encoding '/' and '*' +// - Force encoding of '~' func s3URLEncode(s string) string { spaceCount, hexCount := 0, 0 for i := 0; i < len(s); i++ { diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go index 508c78597..35c95ac2d 100644 --- a/cmd/auth-handler.go +++ b/cmd/auth-handler.go @@ -288,9 +288,10 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in } // Check request auth type verifies the incoming http request -// - validates the request signature -// - validates the policy action if anonymous tests bucket policies if any, -// for authenticated requests validates IAM policies. +// - validates the request signature +// - validates the policy action if anonymous tests bucket policies if any, +// for authenticated requests validates IAM policies. +// // returns APIErrorCode if any to be replied to the client. func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) { _, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName) @@ -298,9 +299,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac } // Check request auth type verifies the incoming http request -// - validates the request signature -// - validates the policy action if anonymous tests bucket policies if any, -// for authenticated requests validates IAM policies. +// - validates the request signature +// - validates the policy action if anonymous tests bucket policies if any, +// for authenticated requests validates IAM policies. +// // returns APIErrorCode if any to be replied to the client. // Additionally returns the accessKey used in the request, and if this request is by an admin. func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) { diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go index a60f52b5c..09895a9fc 100644 --- a/cmd/background-heal-ops.go +++ b/cmd/background-heal-ops.go @@ -26,9 +26,10 @@ import ( ) // healTask represents what to heal along with options -// path: '/' => Heal disk formats along with metadata -// path: 'bucket/' or '/bucket/' => Heal bucket -// path: 'bucket/object' => Heal object +// +// path: '/' => Heal disk formats along with metadata +// path: 'bucket/' or '/bucket/' => Heal bucket +// path: 'bucket/object' => Heal object type healTask struct { bucket string object string diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 29d35bd0c..a24701cb9 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -67,11 +67,16 @@ const ( // - Check if a bucket has an entry in etcd backend // -- If no, make an entry // -- If yes, check if the entry matches local IP check if we -// need to update the entry then proceed to update +// +// need to update the entry then proceed to update +// // -- If yes, check if the IP of entry matches local IP. -// This means entry is for this instance. +// +// This means entry is for this instance. +// // -- If IP of the entry doesn't match, this means entry is -// for another instance. Log an error to console. +// +// for another instance. Log an error to console. func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) { if len(buckets) == 0 { return @@ -227,7 +232,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r * // using the Initiate Multipart Upload request, but has not yet been // completed or aborted. This operation returns at most 1,000 multipart // uploads in the response. -// func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListMultipartUploads") diff --git a/cmd/bucket-listobjects-handlers.go b/cmd/bucket-listobjects-handlers.go index c82513b07..382777f31 100644 --- a/cmd/bucket-listobjects-handlers.go +++ b/cmd/bucket-listobjects-handlers.go @@ -32,9 +32,9 @@ import ( // Validate all the ListObjects query arguments, returns an APIErrorCode // if one of the args do not meet the required conditions. // Special conditions required by MinIO server are as below -// - delimiter if set should be equal to '/', otherwise the request is rejected. -// - marker if set should have a common prefix with 'prefix' param, otherwise -// the request is rejected. +// - delimiter if set should be equal to '/', otherwise the request is rejected. +// - marker if set should have a common prefix with 'prefix' param, otherwise +// the request is rejected. func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode { // Max keys cannot be negative. if maxKeys < 0 { @@ -298,7 +298,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http // This implementation of the GET operation returns some or all (up to 1000) // of the objects in a bucket. You can use the request parameters as selection // criteria to return a subset of the objects in a bucket. -// func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { ctx := newContext(r, w, "ListObjectsV1") diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 172e462db..a7ed3981d 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -1377,7 +1377,6 @@ type ReplicationPool struct { mrfWorkerWg sync.WaitGroup once sync.Once mu sync.Mutex - mrfMutex sync.Mutex } // NewReplicationPool creates a pool of replication workers of specified size diff --git a/cmd/data-usage.go b/cmd/data-usage.go index 1b2146b21..eb6b5fee8 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -56,7 +56,8 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan } // loadPrefixUsageFromBackend returns prefix usages found in passed buckets -// e.g.: /testbucket/prefix => 355601334 +// +// e.g.: /testbucket/prefix => 355601334 func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) { z, ok := objAPI.(*erasureServerPools) if !ok { diff --git a/cmd/dummy-data-generator_test.go b/cmd/dummy-data-generator_test.go index 1038bf45f..90fbc3017 100644 --- a/cmd/dummy-data-generator_test.go +++ b/cmd/dummy-data-generator_test.go @@ -45,10 +45,10 @@ type DummyDataGen struct { // // Given the function: // -// f := func(r io.Reader) string { -// b, _ := ioutil.ReadAll(r) -// return string(b) -// } +// f := func(r io.Reader) string { +// b, _ := ioutil.ReadAll(r) +// return string(b) +// } // // for example, the following is true: // diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index b91bb70e9..07aea1dac 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -231,8 +231,8 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar // // - disks which have all parts specified in the latest xl.meta. // -// - slice of errors about the state of data files on disk - can have -// a not-found error or a hash-mismatch error. +// - slice of errors about the state of data files on disk - can have +// a not-found error or a hash-mismatch error. func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, errs []error, latestMeta FileInfo, bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time, diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go index 3e2dc479b..9fd96530e 100644 --- a/cmd/erasure-server-pool-decom.go +++ b/cmd/erasure-server-pool-decom.go @@ -652,6 +652,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri } // versionsSorter sorts FileInfo slices by version. +// //msgp:ignore versionsSorter type versionsSorter []FileInfo diff --git a/cmd/format-disk-cache.go b/cmd/format-disk-cache.go index 6d202a0c3..c9b32c55c 100644 --- a/cmd/format-disk-cache.go +++ b/cmd/format-disk-cache.go @@ -371,9 +371,10 @@ func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile // migrate cache contents from old cacheFS format to new backend format // new format is flat -// sha(bucket,object)/ <== dir name -// - part.1 <== data -// - cache.json <== metadata +// +// sha(bucket,object)/ <== dir name +// - part.1 <== data +// - cache.json <== metadata func migrateOldCache(ctx context.Context, c *diskCache) error { oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets") cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile) diff --git a/cmd/format-disk-cache_test.go b/cmd/format-disk-cache_test.go index f927558c6..a8a406f0a 100644 --- a/cmd/format-disk-cache_test.go +++ b/cmd/format-disk-cache_test.go @@ -253,13 +253,13 @@ func genFormatCacheInvalidDisksOrder() []*formatCacheV2 { } // Wrapper for calling FormatCache tests - validates -// - valid format -// - unrecognized version number -// - unrecognized format tag -// - unrecognized cache version -// - wrong number of Disks entries -// - invalid This uuid -// - invalid Disks order +// - valid format +// - unrecognized version number +// - unrecognized format tag +// - unrecognized cache version +// - wrong number of Disks entries +// - invalid This uuid +// - invalid Disks order func TestFormatCache(t *testing.T) { formatInputCases := [][]*formatCacheV1{ genFormatCacheValid(), diff --git a/cmd/fs-tree-walk-pool.go b/cmd/fs-tree-walk-pool.go index c85d57960..030803852 100644 --- a/cmd/fs-tree-walk-pool.go +++ b/cmd/fs-tree-walk-pool.go @@ -100,12 +100,12 @@ func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult, // Set - adds a treeWalk to the treeWalkPool. // Also starts a timer go-routine that ends when: -// 1) time.After() expires after t.timeOut seconds. -// The expiration is needed so that the treeWalk go-routine resources are freed after a timeout -// if the S3 client does only partial listing of objects. -// 2) Release() signals the timer go-routine to end on endTimerCh. -// During listing the timer should not timeout and end the treeWalk go-routine, hence the -// timer go-routine should be ended. +// 1. time.After() expires after t.timeOut seconds. +// The expiration is needed so that the treeWalk go-routine resources are freed after a timeout +// if the S3 client does only partial listing of objects. +// 2. Release() signals the timer go-routine to end on endTimerCh. +// During listing the timer should not timeout and end the treeWalk go-routine, hence the +// timer go-routine should be ended. func (t *TreeWalkPool) Set(params listParams, resultCh chan TreeWalkResult, endWalkCh chan struct{}) { t.mu.Lock() defer t.mu.Unlock() diff --git a/cmd/fs-v1-rwpool.go b/cmd/fs-v1-rwpool.go index 462f94a9f..ae714f24a 100644 --- a/cmd/fs-v1-rwpool.go +++ b/cmd/fs-v1-rwpool.go @@ -139,10 +139,10 @@ func (fsi *fsIOPool) Open(path string) (*lock.RLockedFile, error) { } // Write - Attempt to lock the file if it exists, -// - if the file exists. Then we try to get a write lock this -// will block if we can't get a lock perhaps another write -// or read is in progress. Concurrent calls are protected -// by the global namspace lock within the same process. +// - if the file exists. Then we try to get a write lock this +// will block if we can't get a lock perhaps another write +// or read is in progress. Concurrent calls are protected +// by the global namspace lock within the same process. func (fsi *fsIOPool) Write(path string) (wlk *lock.LockedFile, err error) { if err = checkPathLength(path); err != nil { return nil, err diff --git a/cmd/gateway/s3/gateway-s3-chain.go b/cmd/gateway/s3/gateway-s3-chain.go index 2033420b0..daacd349a 100644 --- a/cmd/gateway/s3/gateway-s3-chain.go +++ b/cmd/gateway/s3/gateway-s3-chain.go @@ -37,18 +37,17 @@ import ( // will cache that Provider for all calls to IsExpired(), until Retrieve is // called again after IsExpired() is true. // -// creds := credentials.NewChainCredentials( -// []credentials.Provider{ -// &credentials.EnvAWSS3{}, -// &credentials.EnvMinio{}, -// }) -// -// // Usage of ChainCredentials. -// mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") -// if err != nil { -// log.Fatalln(err) -// } +// creds := credentials.NewChainCredentials( +// []credentials.Provider{ +// &credentials.EnvAWSS3{}, +// &credentials.EnvMinio{}, +// }) // +// // Usage of ChainCredentials. +// mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") +// if err != nil { +// log.Fatalln(err) +// } type Chain struct { Providers []credentials.Provider curr credentials.Provider diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index a96d7543b..15012216e 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -140,21 +140,21 @@ func randString(n int, src rand.Source, prefix string) string { } // Chains all credential types, in the following order: -// - AWS env vars (i.e. AWS_ACCESS_KEY_ID) -// - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) -// - Static credentials provided by user (i.e. MINIO_ROOT_USER/MINIO_ACCESS_KEY) +// - AWS env vars (i.e. AWS_ACCESS_KEY_ID) +// - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) +// - Static credentials provided by user (i.e. MINIO_ROOT_USER/MINIO_ACCESS_KEY) var defaultProviders = []credentials.Provider{ &credentials.EnvAWS{}, &credentials.FileAWSCredentials{}, } // Chains all credential types, in the following order: -// - AWS env vars (i.e. AWS_ACCESS_KEY_ID) -// - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) -// - IAM profile based credentials. (performs an HTTP -// call to a pre-defined endpoint, only valid inside -// configured ec2 instances) -// - Static credentials provided by user (i.e. MINIO_ROOT_USER/MINIO_ACCESS_KEY) +// - AWS env vars (i.e. AWS_ACCESS_KEY_ID) +// - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) +// - IAM profile based credentials. (performs an HTTP +// call to a pre-defined endpoint, only valid inside +// configured ec2 instances) +// - Static credentials provided by user (i.e. MINIO_ROOT_USER/MINIO_ACCESS_KEY) var defaultAWSCredProviders = []credentials.Provider{ &credentials.EnvAWS{}, &credentials.FileAWSCredentials{}, diff --git a/cmd/iam-etcd-store.go b/cmd/iam-etcd-store.go index af42dfaa0..cd10b9364 100644 --- a/cmd/iam-etcd-store.go +++ b/cmd/iam-etcd-store.go @@ -48,10 +48,11 @@ func etcdKvsToSet(prefix string, kvs []*mvccpb.KeyValue) set.StringSet { // Extract path string by stripping off the `prefix` value and the suffix, // value, usually in the following form. -// s := "config/iam/users/foo/config.json" -// prefix := "config/iam/users/" -// suffix := "config.json" -// result is foo +// +// s := "config/iam/users/foo/config.json" +// prefix := "config/iam/users/" +// suffix := "config.json" +// result is foo func extractPathPrefixAndSuffix(s string, prefix string, suffix string) string { return pathClean(strings.TrimSuffix(strings.TrimPrefix(s, prefix), suffix)) } diff --git a/cmd/net.go b/cmd/net.go index f15f68562..63bed5074 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -322,7 +322,8 @@ func isLocalHost(host string, port string, localPort string) (bool, error) { // sameLocalAddrs - returns true if two addresses, even with different // formats, point to the same machine, e.g: -// ':9000' and 'http://localhost:9000/' will return true +// +// ':9000' and 'http://localhost:9000/' will return true func sameLocalAddrs(addr1, addr2 string) (bool, error) { // Extract host & port from given parameters host1, port1, err := extractHostPort(addr1) diff --git a/cmd/notification.go b/cmd/notification.go index 6bffae46c..102ce15cf 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -1125,9 +1125,9 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric // 'freeze' is 'false' would resume all S3 API calls again. // NOTE: once a tenant is frozen either two things needs to // happen before resuming normal operations. -// - Server needs to be restarted 'mc admin service restart' -// - 'freeze' should be set to 'false' for this call -// to resume normal operations. +// - Server needs to be restarted 'mc admin service restart' +// - 'freeze' should be set to 'false' for this call +// to resume normal operations. func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []NotificationPeerErr { serviceSig := serviceUnFreeze if freeze { diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index d02fe5451..69c53b77d 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -33,20 +33,22 @@ var etagRegex = regexp.MustCompile("\"*?([^\"]*?)\"*?$") // Validates the preconditions for CopyObjectPart, returns true if CopyObjectPart // operation should not proceed. Preconditions supported are: -// x-amz-copy-source-if-modified-since -// x-amz-copy-source-if-unmodified-since -// x-amz-copy-source-if-match -// x-amz-copy-source-if-none-match +// +// x-amz-copy-source-if-modified-since +// x-amz-copy-source-if-unmodified-since +// x-amz-copy-source-if-match +// x-amz-copy-source-if-none-match func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool { return checkCopyObjectPreconditions(ctx, w, r, objInfo) } // Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed. // Preconditions supported are: -// x-amz-copy-source-if-modified-since -// x-amz-copy-source-if-unmodified-since -// x-amz-copy-source-if-match -// x-amz-copy-source-if-none-match +// +// x-amz-copy-source-if-modified-since +// x-amz-copy-source-if-unmodified-since +// x-amz-copy-source-if-match +// x-amz-copy-source-if-none-match func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool { // Return false for methods other than GET and HEAD. if r.Method != http.MethodPut { @@ -128,10 +130,11 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r // Validates the preconditions. Returns true if GET/HEAD operation should not proceed. // Preconditions supported are: -// If-Modified-Since -// If-Unmodified-Since -// If-Match -// If-None-Match +// +// If-Modified-Since +// If-Unmodified-Since +// If-Match +// If-None-Match func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, opts ObjectOptions) bool { // Return false for methods other than GET and HEAD. if r.Method != http.MethodGet && r.Method != http.MethodHead { diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index c1c66cec2..e6dd4b869 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -3429,7 +3429,8 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN } // TestAPIPutObjectPartHandler - Tests validate the response of PutObjectPart HTTP handler -// for variety of inputs. +// +// for variety of inputs. func TestAPIPutObjectPartHandler(t *testing.T) { defer DetectTestLeak(t)() ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"}) @@ -3743,7 +3744,8 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin } // TestAPIListObjectPartsHandlerPreSign - Tests validate the response of ListObjectParts HTTP handler -// when signature type of the HTTP request is `Presigned`. +// +// when signature type of the HTTP request is `Presigned`. func TestAPIListObjectPartsHandlerPreSign(t *testing.T) { defer DetectTestLeak(t)() ExecObjectLayerAPITest(t, testAPIListObjectPartsHandlerPreSign, @@ -3832,7 +3834,8 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN } // TestAPIListObjectPartsHandler - Tests validate the response of ListObjectParts HTTP handler -// for variety of success/failure cases. +// +// for variety of success/failure cases. func TestAPIListObjectPartsHandler(t *testing.T) { defer DetectTestLeak(t)() ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"}) diff --git a/cmd/s3-zip-handlers.go b/cmd/s3-zip-handlers.go index 0d948cc36..aaa95a96c 100644 --- a/cmd/s3-zip-handlers.go +++ b/cmd/s3-zip-handlers.go @@ -50,7 +50,8 @@ const ( ) // splitZipExtensionPath splits the S3 path to the zip file and the path inside the zip: -// e.g /path/to/archive.zip/backup-2021/myimage.png => /path/to/archive.zip, backup/myimage.png +// +// e.g /path/to/archive.zip/backup-2021/myimage.png => /path/to/archive.zip, backup/myimage.png func splitZipExtensionPath(input string) (zipPath, object string, err error) { idx := strings.Index(input, archivePattern) if idx < 0 { diff --git a/cmd/signature-v2.go b/cmd/signature-v2.go index 7f789d268..dc31cae0d 100644 --- a/cmd/signature-v2.go +++ b/cmd/signature-v2.go @@ -107,7 +107,8 @@ func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) } // doesPresignV2SignatureMatch - Verify query headers with presigned signature -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth +// // returns ErrNone if matches. S3 errors otherwise. func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode { // r.RequestURI will have raw encoded URI as sent by the client. diff --git a/cmd/signature-v4-parser.go b/cmd/signature-v4-parser.go index 9e28f60cf..366044628 100644 --- a/cmd/signature-v4-parser.go +++ b/cmd/signature-v4-parser.go @@ -171,12 +171,12 @@ type preSignValues struct { // Parses signature version '4' query string of the following form. // -// querystring = X-Amz-Algorithm=algorithm -// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope) -// querystring += &X-Amz-Date=date -// querystring += &X-Amz-Expires=timeout interval -// querystring += &X-Amz-SignedHeaders=signed_headers -// querystring += &X-Amz-Signature=signature +// querystring = X-Amz-Algorithm=algorithm +// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope) +// querystring += &X-Amz-Date=date +// querystring += &X-Amz-Expires=timeout interval +// querystring += &X-Amz-SignedHeaders=signed_headers +// querystring += &X-Amz-Signature=signature // // verifies if any of the necessary query params are missing in the presigned request. func doesV4PresignParamsExist(query url.Values) APIErrorCode { @@ -251,9 +251,8 @@ func parsePreSignV4(query url.Values, region string, stype serviceType) (psv pre // Parses signature version '4' header of the following form. // -// Authorization: algorithm Credential=accessKeyID/credScope, \ -// SignedHeaders=signedHeaders, Signature=signature -// +// Authorization: algorithm Credential=accessKeyID/credScope, \ +// SignedHeaders=signedHeaders, Signature=signature func parseSignV4(v4Auth string, region string, stype serviceType) (sv signValues, aec APIErrorCode) { // credElement is fetched first to skip replacing the space in access key. credElement := strings.TrimPrefix(strings.Split(strings.TrimSpace(v4Auth), ",")[0], signV4Algorithm) diff --git a/cmd/signature-v4-parser_test.go b/cmd/signature-v4-parser_test.go index 8c32a4f0a..427605d59 100644 --- a/cmd/signature-v4-parser_test.go +++ b/cmd/signature-v4-parser_test.go @@ -82,10 +82,11 @@ func validateCredentialfields(t *testing.T, testNum int, expectedCredentials cre // A valid format of creadential should be of the following format. // Credential = accessKey + SlashSeparator+ scope // where scope = string.Join([]string{ currTime.Format(yyyymmdd), -// globalMinioDefaultRegion, -// "s3", -// "aws4_request", -// },SlashSeparator) +// +// globalMinioDefaultRegion, +// "s3", +// "aws4_request", +// },SlashSeparator) func TestParseCredentialHeader(t *testing.T) { sampleTimeStr := UTCNow().Format(yyyymmdd) diff --git a/cmd/signature-v4.go b/cmd/signature-v4.go index 740aa330c..fc847eb62 100644 --- a/cmd/signature-v4.go +++ b/cmd/signature-v4.go @@ -95,13 +95,13 @@ func getSignedHeaders(signedHeaders http.Header) string { // getCanonicalRequest generate a canonical request of style // // canonicalRequest = -// \n -// \n -// \n -// \n -// \n -// // +// \n +// \n +// \n +// \n +// \n +// func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, urlPath, method string) string { rawQuery := strings.ReplaceAll(queryStr, "+", "%20") encodedPath := s3utils.EncodePath(urlPath) @@ -169,7 +169,8 @@ func compareSignatureV4(sig1, sig2 string) bool { } // doesPolicySignatureMatch - Verify query headers with post policy -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// // returns ErrNone if the signature matches. func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIErrorCode) { // Server region. @@ -203,7 +204,8 @@ func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIEr } // doesPresignedSignatureMatch - Verify query headers with presigned signature -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// // returns ErrNone if the signature matches. func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode { // Copy request @@ -329,7 +331,8 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s } // doesSignatureMatch - Verify authorization header with calculated header in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +// // returns ErrNone if signature matches. func doesSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode { // Copy request. diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index 48fa3ef1a..3920daa63 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -22,6 +22,7 @@ import ( ) // DeleteOptions represents the disk level delete options available for the APIs +// //msgp:ignore DeleteOptions type DeleteOptions struct { Recursive bool @@ -32,8 +33,11 @@ type DeleteOptions struct { // DiskInfo is an extended type which returns current // disk usage per path. -//msgp:tuple DiskInfo // The above means that any added/deleted fields are incompatible. +// +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple DiskInfo type DiskInfo struct { Total uint64 Free uint64 @@ -65,8 +69,11 @@ type DiskMetrics struct { type VolsInfo []VolInfo // VolInfo - represents volume stat information. -//msgp:tuple VolInfo // The above means that any added/deleted fields are incompatible. +// +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple VolInfo type VolInfo struct { // Name of the volume. Name string @@ -77,6 +84,8 @@ type VolInfo struct { // FilesInfo represent a list of files, additionally // indicates if the list is last. +// +//msgp:tuple FileInfo type FilesInfo struct { Files []FileInfo IsTruncated bool @@ -91,8 +100,11 @@ func (f FileInfoVersions) Size() (size int64) { } // FileInfoVersions represent a list of versions for a given file. -//msgp:tuple FileInfoVersions // The above means that any added/deleted fields are incompatible. +// +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple FileInfoVersions type FileInfoVersions struct { // Name of the volume. Volume string `msg:"v,omitempty"` @@ -136,7 +148,6 @@ type RawFileInfo struct { } // FileInfo - represents file stat information. -//msgp:tuple FileInfo // The above means that any added/deleted fields are incompatible. // Make sure to bump the internode version at storage-rest-common.go type FileInfo struct { @@ -235,10 +246,10 @@ func (fi FileInfo) Equals(ofi FileInfo) (ok bool) { } // GetDataDir returns an expected dataDir given FileInfo -// - deleteMarker returns "delete-marker" -// - returns "legacy" if FileInfo is XLV1 and DataDir is -// empty, returns DataDir otherwise -// - returns "dataDir" +// - deleteMarker returns "delete-marker" +// - returns "legacy" if FileInfo is XLV1 and DataDir is +// empty, returns DataDir otherwise +// - returns "dataDir" func (fi FileInfo) GetDataDir() string { if fi.Deleted { return "delete-marker" diff --git a/cmd/streaming-signature-v4.go b/cmd/streaming-signature-v4.go index 9465145f7..6adfcfe5f 100644 --- a/cmd/streaming-signature-v4.go +++ b/cmd/streaming-signature-v4.go @@ -63,7 +63,8 @@ func getChunkSignature(cred auth.Credentials, seedSignature string, region strin } // calculateSeedSignature - Calculate seed signature in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +// // returns signature, error otherwise if the signature mismatches or any other // error while parsing and validating. func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) { @@ -195,7 +196,8 @@ func (cr *s3ChunkedReader) Close() (err error) { // Now, we read one chunk from the underlying reader. // A chunk has the following format: -// + ";chunk-signature=" + + "\r\n" + + "\r\n" +// +// + ";chunk-signature=" + + "\r\n" + + "\r\n" // // First, we read the chunk size but fail if it is larger // than 16 MiB. We must not accept arbitrary large chunks. @@ -414,7 +416,8 @@ const s3ChunkSignatureStr = ";chunk-signature=" // parses3ChunkExtension removes any s3 specific chunk-extension from buf. // For example, -// "10000;chunk-signature=..." => "10000", "chunk-signature=..." +// +// "10000;chunk-signature=..." => "10000", "chunk-signature=..." func parseS3ChunkExtension(buf []byte) ([]byte, []byte) { buf = trimTrailingWhitespace(buf) semi := bytes.Index(buf, []byte(s3ChunkSignatureStr)) diff --git a/cmd/sts-handlers.go b/cmd/sts-handlers.go index 5ee154f08..c9de8ab99 100644 --- a/cmd/sts-handlers.go +++ b/cmd/sts-handlers.go @@ -522,7 +522,8 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ // Connect-compatible identity provider. // // Eg:- -// $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken= +// +// $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken= func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) { sts.AssumeRoleWithSSO(w, r) } @@ -531,7 +532,8 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h // OAuth2.0 client credential grants. // // Eg:- -// $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token= +// +// $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token= func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) { sts.AssumeRoleWithSSO(w, r) } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index b33654299..ea054e250 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -127,19 +127,17 @@ func TestMain(m *testing.M) { // concurrency level for certain parallel tests. const testConcurrencyLevel = 10 -// // Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 // -// User-Agent: +// User-Agent: // -// This is ignored from signing because signing this causes problems with generating pre-signed URLs -// (that are executed by other agents) or when customers pass requests through proxies, which may -// modify the user-agent. +// This is ignored from signing because signing this causes problems with generating pre-signed URLs +// (that are executed by other agents) or when customers pass requests through proxies, which may +// modify the user-agent. // -// Authorization: -// -// Is skipped for obvious reasons +// Authorization: // +// Is skipped for obvious reasons var ignoredHeaders = map[string]bool{ "Authorization": true, "User-Agent": true, @@ -302,8 +300,9 @@ func isSameType(obj1, obj2 interface{}) bool { // TestServer encapsulates an instantiation of a MinIO instance with a temporary backend. // Example usage: -// s := StartTestServer(t,"Erasure") -// defer s.Stop() +// +// s := StartTestServer(t,"Erasure") +// defer s.Stop() type TestServer struct { Root string Disks EndpointServerPools @@ -1563,11 +1562,14 @@ func prepareTestBackend(ctx context.Context, instanceType string) (ObjectLayer, // response for anonymous/unsigned and unknown signature type HTTP request. // Here is the brief description of some of the arguments to the function below. -// apiRouter - http.Handler with the relevant API endPoint (API endPoint under test) registered. -// anonReq - unsigned *http.Request to invoke the handler's response for anonymous requests. -// policyFunc - function to return bucketPolicy statement which would permit the anonymous request to be served. +// +// apiRouter - http.Handler with the relevant API endPoint (API endPoint under test) registered. +// anonReq - unsigned *http.Request to invoke the handler's response for anonymous requests. +// policyFunc - function to return bucketPolicy statement which would permit the anonymous request to be served. +// // The test works in 2 steps, here is the description of the steps. -// STEP 1: Call the handler with the unsigned HTTP request (anonReq), assert for the `ErrAccessDenied` error response. +// +// STEP 1: Call the handler with the unsigned HTTP request (anonReq), assert for the `ErrAccessDenied` error response. func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketName, objectName, instanceType string, apiRouter http.Handler, anonReq *http.Request, bucketPolicy *policy.Policy, ) { diff --git a/cmd/tier-sweeper.go b/cmd/tier-sweeper.go index 8553fc205..f70dce234 100644 --- a/cmd/tier-sweeper.go +++ b/cmd/tier-sweeper.go @@ -27,15 +27,17 @@ import ( // // Perform a ObjectLayer.GetObjectInfo to fetch object version information // goiOpts := os.GetOpts() // gerr := objAPI.GetObjectInfo(ctx, bucket, object, goiOpts) -// if gerr == nil { -// os.SetTransitionState(goi) -// } +// +// if gerr == nil { +// os.SetTransitionState(goi) +// } // // // After the overwriting object operation is complete. -// if jentry, ok := os.ShouldRemoveRemoteObject(); ok { -// err := globalTierJournal.AddEntry(jentry) -// logger.LogIf(ctx, err) -// } +// +// if jentry, ok := os.ShouldRemoveRemoteObject(); ok { +// err := globalTierJournal.AddEntry(jentry) +// logger.LogIf(ctx, err) +// } type objSweeper struct { Object string Bucket string diff --git a/cmd/tree-walk.go b/cmd/tree-walk.go index aa559990f..1636705b5 100644 --- a/cmd/tree-walk.go +++ b/cmd/tree-walk.go @@ -51,8 +51,10 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string { // we need to remove this trailing "/" for objects and retain "/" for prefixes before // sorting because the trailing "/" can affect the sorting results for certain cases. // Ex. lets say entries = ["a-b/", "a/"] and both are objects. -// sorting with out trailing "/" = ["a", "a-b"] -// sorting with trailing "/" = ["a-b/", "a/"] +// +// sorting with out trailing "/" = ["a", "a-b"] +// sorting with trailing "/" = ["a-b/", "a/"] +// // Hence if entries[] does not have a case like the above example then isLeaf() check // can be delayed till the entry is pushed into the TreeWalkResult channel. // delayIsLeafCheck() returns true if isLeaf can be delayed or false if @@ -86,10 +88,10 @@ type ListDirFunc func(bucket, prefixDir, prefixEntry string) (emptyDir bool, ent // IsLeafFunc - A function isLeaf of type isLeafFunc is used to detect if an // entry is a leaf entry. There are 2 scenarios where isLeaf should behave // differently depending on the backend: -// 1. FS backend object listing - isLeaf is true if the entry -// has no trailing "/" -// 2. Erasure backend object listing - isLeaf is true if the entry -// is a directory and contains xl.meta +// 1. FS backend object listing - isLeaf is true if the entry +// has no trailing "/" +// 2. Erasure backend object listing - isLeaf is true if the entry +// is a directory and contains xl.meta type IsLeafFunc func(string, string) bool // IsLeafDirFunc - A function isLeafDir of type isLeafDirFunc is used to detect diff --git a/cmd/update.go b/cmd/update.go index 4e4292edb..ff85eeaac 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -123,8 +123,7 @@ func GetCurrentReleaseTime() (releaseTime time.Time, err error) { // // https://github.com/moby/moby/blob/master/daemon/initlayer/setup_unix.go#L25 // -// "/.dockerenv": "file", -// +// "/.dockerenv": "file", func IsDocker() bool { if !globalIsCICD { _, err := os.Stat("/.dockerenv") @@ -220,7 +219,7 @@ func IsPCFTile() bool { // DO NOT CHANGE USER AGENT STYLE. // The style should be // -// MinIO (; [; ][; dcos][; kubernetes][; docker][; source]) MinIO/ MinIO/ MinIO/ [MinIO/universe-] [MinIO/helm-] +// MinIO (; [; ][; dcos][; kubernetes][; docker][; source]) MinIO/ MinIO/ MinIO/ [MinIO/universe-] [MinIO/helm-] // // Any change here should be discussed by opening an issue at // https://github.com/minio/minio/issues. diff --git a/internal/bucket/lifecycle/lifecycle.go b/internal/bucket/lifecycle/lifecycle.go index 9e70c2ea1..2bfd5399c 100644 --- a/internal/bucket/lifecycle/lifecycle.go +++ b/internal/bucket/lifecycle/lifecycle.go @@ -390,8 +390,9 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action { // ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime. // The expected transition or restore time is always a midnight time following the the object // modification time plus the number of transition/restore days. -// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should -// transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT` +// +// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should +// transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT` func ExpectedExpiryTime(modTime time.Time, days int) time.Time { if days == 0 { return modTime diff --git a/internal/config/heal/heal.go b/internal/config/heal/heal.go index 0ab84bb16..6c9f2ce2f 100644 --- a/internal/config/heal/heal.go +++ b/internal/config/heal/heal.go @@ -60,7 +60,9 @@ type Config struct { // BitrotScanCycle returns the configured cycle for the scanner healing // -1 for not enabled -// 0 for contiunous bitrot scanning +// +// 0 for contiunous bitrot scanning +// // >0 interval duration between cycles func (opts Config) BitrotScanCycle() (d time.Duration) { configMutex.RLock() diff --git a/internal/config/storageclass/storage-class.go b/internal/config/storageclass/storage-class.go index 8527647ab..7ada9c923 100644 --- a/internal/config/storageclass/storage-class.go +++ b/internal/config/storageclass/storage-class.go @@ -216,10 +216,13 @@ func validateParity(ssParity, rrsParity, setDriveCount int) (err error) { // // -- if input storage class is empty then standard is assumed // -- if input is RRS but RRS is not configured default '2' parity -// for RRS is assumed +// +// for RRS is assumed +// // -- if input is STANDARD but STANDARD is not configured '0' parity -// is returned, the caller is expected to choose the right parity -// at that point. +// +// is returned, the caller is expected to choose the right parity +// at that point. func (sCfg Config) GetParityForSC(sc string) (parity int) { ConfigLock.RLock() defer ConfigLock.RUnlock() diff --git a/internal/crypto/doc.go b/internal/crypto/doc.go index e8f7b8a11..eac2e4acc 100644 --- a/internal/crypto/doc.go +++ b/internal/crypto/doc.go @@ -25,32 +25,30 @@ // with an unique key-encryption-key. Given the correct key-encryption-key the // sealed 'ObjectKey' can be unsealed and the object can be decrypted. // -// // ## SSE-C // // SSE-C computes the key-encryption-key from the client-provided key, an // initialization vector (IV) and the bucket/object path. // -// 1. Encrypt: -// Input: ClientKey, bucket, object, metadata, object_data -// - IV := Random({0,1}²⁵⁶) -// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶)) -// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) -// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) -// - metadata <- IV -// - metadata <- SealedKey -// Output: enc_object_data, metadata -// -// 2. Decrypt: -// Input: ClientKey, bucket, object, metadata, enc_object_data -// - IV <- metadata -// - SealedKey <- metadata -// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) -// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) -// Output: object_data +// 1. Encrypt: +// Input: ClientKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata // +// 2. Decrypt: +// Input: ClientKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data // // ## SSE-S3 // @@ -63,57 +61,57 @@ // SSE-S3 with a single master key works as SSE-C where the master key is // used as the client-provided key. // -// 1. Encrypt: -// Input: MasterKey, bucket, object, metadata, object_data -// - IV := Random({0,1}²⁵⁶) -// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶)) -// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) -// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) -// - metadata <- IV -// - metadata <- SealedKey -// Output: enc_object_data, metadata -// -// 2. Decrypt: -// Input: MasterKey, bucket, object, metadata, enc_object_data -// - IV <- metadata -// - SealedKey <- metadata -// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) -// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) -// Output: object_data +// 1. Encrypt: +// Input: MasterKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata // +// 2. Decrypt: +// Input: MasterKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data // // ### SSE-S3 and KMS // // SSE-S3 requires that the KMS provides two functions: -// 1. Generate(KeyID) -> (Key, EncKey) -// 2. Unseal(KeyID, EncKey) -> Key // -// 1. Encrypt: -// Input: KeyID, bucket, object, metadata, object_data -// - Key, EncKey := Generate(KeyID) -// - IV := Random({0,1}²⁵⁶) -// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶)) -// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) -// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) -// - metadata <- IV -// - metadata <- KeyID -// - metadata <- EncKey -// - metadata <- SealedKey -// Output: enc_object_data, metadata +// 1. Generate(KeyID) -> (Key, EncKey) // -// 2. Decrypt: -// Input: bucket, object, metadata, enc_object_data -// - KeyID <- metadata -// - EncKey <- metadata -// - IV <- metadata -// - SealedKey <- metadata -// - Key := Unseal(KeyID, EncKey) -// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) -// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) -// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) -// Output: object_data +// 2. Unseal(KeyID, EncKey) -> Key // +// 1. Encrypt: +// Input: KeyID, bucket, object, metadata, object_data +// - Key, EncKey := Generate(KeyID) +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- KeyID +// - metadata <- EncKey +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: bucket, object, metadata, enc_object_data +// - KeyID <- metadata +// - EncKey <- metadata +// - IV <- metadata +// - SealedKey <- metadata +// - Key := Unseal(KeyID, EncKey) +// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data package crypto diff --git a/internal/crypto/sse.go b/internal/crypto/sse.go index 60c0c18a5..9f4eeff68 100644 --- a/internal/crypto/sse.go +++ b/internal/crypto/sse.go @@ -43,9 +43,9 @@ const ( ) // Type represents an AWS SSE type: -// • SSE-C -// • SSE-S3 -// • SSE-KMS +// - SSE-C +// - SSE-S3 +// - SSE-KMS type Type interface { fmt.Stringer diff --git a/internal/dsync/dsync_test.go b/internal/dsync/dsync_test.go index c717b2db1..8e6364b23 100644 --- a/internal/dsync/dsync_test.go +++ b/internal/dsync/dsync_test.go @@ -206,7 +206,6 @@ func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) { } // Test refreshing lock - refresh should always return true -// func TestSuccessfulLockRefresh(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") diff --git a/internal/etag/etag.go b/internal/etag/etag.go index 2e2d91f8d..63f182c52 100644 --- a/internal/etag/etag.go +++ b/internal/etag/etag.go @@ -24,35 +24,35 @@ // In general, an S3 ETag is an MD5 checksum of the object // content. However, there are many exceptions to this rule. // -// -// Single-part Upload +// # Single-part Upload // // In case of a basic single-part PUT operation - without server // side encryption or object compression - the ETag of an object // is its content MD5. // -// -// Multi-part Upload +// # Multi-part Upload // // The ETag of an object does not correspond to its content MD5 // when the object is uploaded in multiple parts via the S3 // multipart API. Instead, S3 first computes a MD5 of each part: -// e1 := MD5(part-1) -// e2 := MD5(part-2) -// ... -// eN := MD5(part-N) +// +// e1 := MD5(part-1) +// e2 := MD5(part-2) +// ... +// eN := MD5(part-N) // // Then, the ETag of the object is computed as MD5 of all individual // part checksums. S3 also encodes the number of parts into the ETag // by appending a - at the end: -// ETag := MD5(e1 || e2 || e3 ... || eN) || -N // -// For example: ceb8853ddc5086cc4ab9e149f8f09c88-5 +// ETag := MD5(e1 || e2 || e3 ... || eN) || -N +// +// For example: ceb8853ddc5086cc4ab9e149f8f09c88-5 // // However, this scheme is only used for multipart objects that are // not encrypted. // -// Server-side Encryption +// # Server-side Encryption // // S3 specifies three types of server-side-encryption - SSE-C, SSE-S3 // and SSE-KMS - with different semantics w.r.t. ETags. @@ -75,12 +75,12 @@ // in case of SSE-C or SSE-KMS except that the ETag is well-formed. // // To put all of this into a simple rule: -// SSE-S3 : ETag == MD5 -// SSE-C : ETag != MD5 -// SSE-KMS: ETag != MD5 // +// SSE-S3 : ETag == MD5 +// SSE-C : ETag != MD5 +// SSE-KMS: ETag != MD5 // -// Encrypted ETags +// # Encrypted ETags // // An S3 implementation has to remember the content MD5 of objects // in case of SSE-S3. However, storing the ETag of an encrypted @@ -94,8 +94,7 @@ // encryption schemes. Such an ETag must be decrypted before sent to an // S3 client. // -// -// S3 Clients +// # S3 Clients // // There are many different S3 client implementations. Most of them // access the ETag by looking for the HTTP response header key "Etag". @@ -206,27 +205,28 @@ func (e ETag) Parts() int { // ETag. // // In general, a caller has to distinguish the following cases: -// - The object is a multipart object. In this case, -// Format returns the ETag unmodified. -// - The object is a SSE-KMS or SSE-C encrypted single- -// part object. In this case, Format returns the last -// 16 bytes of the encrypted ETag which will be a random -// value. -// - The object is a SSE-S3 encrypted single-part object. -// In this case, the caller has to decrypt the ETag first -// before calling Format. -// S3 clients expect that the ETag of an SSE-S3 encrypted -// single-part object is equal to the object's content MD5. -// Formatting the SSE-S3 ETag before decryption will result -// in a random-looking ETag which an S3 client will not accept. +// - The object is a multipart object. In this case, +// Format returns the ETag unmodified. +// - The object is a SSE-KMS or SSE-C encrypted single- +// part object. In this case, Format returns the last +// 16 bytes of the encrypted ETag which will be a random +// value. +// - The object is a SSE-S3 encrypted single-part object. +// In this case, the caller has to decrypt the ETag first +// before calling Format. +// S3 clients expect that the ETag of an SSE-S3 encrypted +// single-part object is equal to the object's content MD5. +// Formatting the SSE-S3 ETag before decryption will result +// in a random-looking ETag which an S3 client will not accept. // // Hence, a caller has to check: -// if method == SSE-S3 { -// ETag, err := Decrypt(key, ETag) -// if err != nil { -// } -// } -// ETag = ETag.Format() +// +// if method == SSE-S3 { +// ETag, err := Decrypt(key, ETag) +// if err != nil { +// } +// } +// ETag = ETag.Format() func (e ETag) Format() ETag { if !e.IsEncrypted() { return e @@ -359,8 +359,8 @@ func Parse(s string) (ETag, error) { // parse parse s as an S3 ETag, returning the result. // It operates in one of two modes: -// - strict -// - non-strict +// - strict +// - non-strict // // In strict mode, parse only accepts ETags that // are AWS S3 compatible. In particular, an AWS diff --git a/internal/etag/reader.go b/internal/etag/reader.go index 2ee59a69c..3dd9c41c9 100644 --- a/internal/etag/reader.go +++ b/internal/etag/reader.go @@ -56,15 +56,14 @@ func (r wrapReader) ETag() ETag { // It is mainly used to provide a high-level io.Reader // access to the ETag computed by a low-level io.Reader: // -// content := etag.NewReader(r.Body, nil) +// content := etag.NewReader(r.Body, nil) // -// compressedContent := Compress(content) -// encryptedContent := Encrypt(compressedContent) -// -// // Now, we need an io.Reader that can access -// // the ETag computed over the content. -// reader := etag.Wrap(encryptedContent, content) +// compressedContent := Compress(content) +// encryptedContent := Encrypt(compressedContent) // +// // Now, we need an io.Reader that can access +// // the ETag computed over the content. +// reader := etag.Wrap(encryptedContent, content) func Wrap(wrapped, content io.Reader) io.Reader { if t, ok := content.(Tagger); ok { return wrapReader{ diff --git a/internal/handlers/forwarder.go b/internal/handlers/forwarder.go index b4a77223c..12b2d69eb 100644 --- a/internal/handlers/forwarder.go +++ b/internal/handlers/forwarder.go @@ -72,7 +72,8 @@ func (f *Forwarder) ServeHTTP(w http.ResponseWriter, inReq *http.Request) { } // customErrHandler is originally implemented to avoid having the following error -// `http: proxy error: context canceled` printed by Golang +// +// `http: proxy error: context canceled` printed by Golang func (f *Forwarder) customErrHandler(w http.ResponseWriter, r *http.Request, err error) { if f.Logger != nil && err != context.Canceled { f.Logger(err) diff --git a/internal/kms/single-key.go b/internal/kms/single-key.go index 687e37a77..95b6e9829 100644 --- a/internal/kms/single-key.go +++ b/internal/kms/single-key.go @@ -39,7 +39,8 @@ import ( // Parse parses s as single-key KMS. The given string // is expected to have the following format: -// : +// +// : // // The returned KMS implementation uses the parsed // key ID and key to derive new DEKs and decrypt ciphertext. diff --git a/internal/logger/targets.go b/internal/logger/targets.go index 9f616e001..c64166dd0 100644 --- a/internal/logger/targets.go +++ b/internal/logger/targets.go @@ -27,7 +27,8 @@ import ( // Target is the entity that we will receive // a single log entry and Send it to the log target -// e.g. Send the log to a http server +// +// e.g. Send the log to a http server type Target interface { String() string Endpoint() string @@ -126,8 +127,9 @@ func initKafkaTargets(cfgMap map[string]kafka.Config) (tgts []Target, err error) } // Split targets into two groups: -// group1 contains all targets of type t -// group2 contains the remaining targets +// +// group1 contains all targets of type t +// group2 contains the remaining targets func splitTargets(targets []Target, t types.TargetType) (group1 []Target, group2 []Target) { for _, target := range targets { if target.Type() == t { diff --git a/internal/s3select/message.go b/internal/s3select/message.go index 61931920f..ecac40a5b 100644 --- a/internal/s3select/message.go +++ b/internal/s3select/message.go @@ -128,9 +128,9 @@ var progressHeader = []byte{ // // Payload specification: // Progress message payload is an XML document containing information about the progress of a request. -// * BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed). -// * BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed). -// * BytesReturned => Current number of bytes of records payload data returned by S3. +// - BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed). +// - BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed). +// - BytesReturned => Current number of bytes of records payload data returned by S3. // // For uncompressed files, BytesScanned and BytesProcessed are equal. // @@ -138,11 +138,12 @@ var progressHeader = []byte{ // // // -// 512 -// 1024 -// 1024 -// // +// 512 +// 1024 +// 1024 +// +// func newProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte { payload := []byte(`` + strconv.FormatInt(bytesScanned, 10) + `` + @@ -167,9 +168,9 @@ var statsHeader = []byte{ // // Payload specification: // Stats message payload is an XML document containing information about a request's stats when processing is complete. -// * BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed). -// * BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed). -// * BytesReturned => Total number of bytes of records payload data returned by S3. +// - BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed). +// - BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed). +// - BytesReturned => Total number of bytes of records payload data returned by S3. // // For uncompressed files, BytesScanned and BytesProcessed are equal. // @@ -177,9 +178,11 @@ var statsHeader = []byte{ // // // -// 512 -// 1024 -// 1024 +// +// 512 +// 1024 +// 1024 +// // func newStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte { payload := []byte(`` + diff --git a/internal/smart/types.go b/internal/smart/types.go index 432b3d089..fdf2dab50 100644 --- a/internal/smart/types.go +++ b/internal/smart/types.go @@ -18,6 +18,7 @@ package smart // Defined in +// //nolint:structcheck,deadcode type nvmePassthruCommand struct { opcode uint8 @@ -138,6 +139,7 @@ type nvmeSMARTLog struct { } // 512 bytes // NVMeDevice represents drive data about NVMe drives +// //nolint:structcheck type NVMeDevice struct { Name string