mirror of
https://github.com/minio/minio.git
synced 2025-03-06 00:30:09 -05:00
tests: Do not allow forced type asserts (#20905)
This commit is contained in:
parent
aeabac9181
commit
90f5e1e5f6
@ -21,16 +21,25 @@ linters:
|
|||||||
- misspell
|
- misspell
|
||||||
- revive
|
- revive
|
||||||
- staticcheck
|
- staticcheck
|
||||||
- tenv
|
|
||||||
- typecheck
|
- typecheck
|
||||||
- unconvert
|
- unconvert
|
||||||
- unused
|
- unused
|
||||||
|
- usetesting
|
||||||
|
- forcetypeassert
|
||||||
|
- whitespace
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
exclude-use-default: false
|
exclude-use-default: false
|
||||||
|
max-issues-per-linter: 100
|
||||||
|
max-same-issues: 100
|
||||||
exclude:
|
exclude:
|
||||||
- "empty-block:"
|
- "empty-block:"
|
||||||
- "unused-parameter:"
|
- "unused-parameter:"
|
||||||
- "dot-imports:"
|
- "dot-imports:"
|
||||||
- should have a package comment
|
- should have a package comment
|
||||||
- error strings should not be capitalized or end with punctuation or a newline
|
- error strings should not be capitalized or end with punctuation or a newline
|
||||||
|
exclude-rules:
|
||||||
|
# Exclude some linters from running on tests files.
|
||||||
|
- path: _test\.go
|
||||||
|
linters:
|
||||||
|
- forcetypeassert
|
||||||
|
@ -38,6 +38,7 @@ import (
|
|||||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
"github.com/minio/minio/internal/bucket/versioning"
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/minio/pkg/v3/policy"
|
||||||
@ -980,7 +981,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
rpt.SetStatus(bucket, "", err)
|
rpt.SetStatus(bucket, "", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||||
@ -1039,7 +1039,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
if len(diffCh) == 0 {
|
if len(diffCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case <-keepAliveTicker.C:
|
case <-keepAliveTicker.C:
|
||||||
if len(diffCh) > 0 {
|
if len(diffCh) > 0 {
|
||||||
@ -1048,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
|||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -1098,7 +1098,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
|||||||
}
|
}
|
||||||
if len(mrfCh) == 0 {
|
if len(mrfCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case <-keepAliveTicker.C:
|
case <-keepAliveTicker.C:
|
||||||
if len(mrfCh) > 0 {
|
if len(mrfCh) > 0 {
|
||||||
@ -1107,7 +1107,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
|||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -125,7 +125,6 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||||
|
|
||||||
var validationErr ldap.Validation
|
var validationErr ldap.Validation
|
||||||
if errors.As(err, &validationErr) {
|
if errors.As(err, &validationErr) {
|
||||||
// If we got an LDAP validation error, we need to send appropriate
|
// If we got an LDAP validation error, we need to send appropriate
|
||||||
@ -416,7 +415,6 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||||
|
|
||||||
var validationErr ldap.Validation
|
var validationErr ldap.Validation
|
||||||
if errors.As(err, &validationErr) {
|
if errors.As(err, &validationErr) {
|
||||||
// If we got an LDAP validation error, we need to send appropriate
|
// If we got an LDAP validation error, we need to send appropriate
|
||||||
|
@ -1487,8 +1487,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
@ -2279,7 +2279,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
|
|
||||||
// import policies first
|
// import policies first
|
||||||
{
|
{
|
||||||
|
|
||||||
f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile))
|
f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile))
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, os.ErrNotExist):
|
case errors.Is(err, os.ErrNotExist):
|
||||||
@ -2362,7 +2361,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
|
|||||||
} else {
|
} else {
|
||||||
added.Users = append(added.Users, accessKey)
|
added.Users = append(added.Users, accessKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -829,7 +829,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Flush before waiting for next...
|
// Flush before waiting for next...
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
@ -1359,7 +1359,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case hr := <-respCh:
|
case hr := <-respCh:
|
||||||
switch hr.apiErr {
|
switch hr.apiErr {
|
||||||
case noError:
|
case noError:
|
||||||
@ -1367,7 +1367,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
if _, err := w.Write(hr.respBytes); err != nil {
|
if _, err := w.Write(hr.respBytes); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
} else {
|
} else {
|
||||||
writeSuccessResponseJSON(w, hr.respBytes)
|
writeSuccessResponseJSON(w, hr.respBytes)
|
||||||
}
|
}
|
||||||
@ -1394,7 +1394,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
if _, err := w.Write(errorRespJSON); err != nil {
|
if _, err := w.Write(errorRespJSON); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
break forLoop
|
break forLoop
|
||||||
}
|
}
|
||||||
@ -1611,7 +1611,6 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
|
|||||||
if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
|
if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
@ -1840,7 +1839,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case result, ok := <-ch:
|
case result, ok := <-ch:
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@ -1849,7 +1848,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
prevResult = result
|
prevResult = result
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1958,7 +1957,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
|||||||
if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
|
if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case result, ok := <-ch:
|
case result, ok := <-ch:
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@ -1966,7 +1965,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
|
|||||||
if err := enc.Encode(result); err != nil {
|
if err := enc.Encode(result); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2083,7 +2082,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
grid.PutByteBuffer(entry)
|
grid.PutByteBuffer(entry)
|
||||||
if len(traceCh) == 0 {
|
if len(traceCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case <-keepAliveTicker.C:
|
case <-keepAliveTicker.C:
|
||||||
if len(traceCh) > 0 {
|
if len(traceCh) > 0 {
|
||||||
@ -2092,7 +2091,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -2184,7 +2183,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
grid.PutByteBuffer(log)
|
grid.PutByteBuffer(log)
|
||||||
if len(logCh) == 0 {
|
if len(logCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case <-keepAliveTicker.C:
|
case <-keepAliveTicker.C:
|
||||||
if len(logCh) > 0 {
|
if len(logCh) > 0 {
|
||||||
@ -2193,7 +2192,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -2963,13 +2962,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
|||||||
}
|
}
|
||||||
if len(healthInfoCh) == 0 {
|
if len(healthInfoCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-healthCtx.Done():
|
case <-healthCtx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -520,7 +520,6 @@ func cleanReservedKeys(metadata map[string]string) map[string]string {
|
|||||||
}
|
}
|
||||||
case crypto.SSEC:
|
case crypto.SSEC:
|
||||||
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var toRemove []string
|
var toRemove []string
|
||||||
|
@ -162,7 +162,6 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
|||||||
s3Err := ErrAccessDenied
|
s3Err := ErrAccessDenied
|
||||||
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
||||||
getRequestAuthType(r) == authTypeSigned {
|
getRequestAuthType(r) == authTypeSigned {
|
||||||
|
|
||||||
// Get credential information from the request.
|
// Get credential information from the request.
|
||||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
|
@ -195,8 +195,8 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
||||||
for _, kv := range ef.Metadata {
|
for _, kv := range ef.Metadata {
|
||||||
// Object (version) must match all x-amz-meta and
|
// Object (version) must match all x-amz-meta and
|
||||||
|
@ -1450,7 +1450,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
|
|||||||
cred = r.Source.Creds
|
cred = r.Source.Creds
|
||||||
remoteBkt = r.Source.Bucket
|
remoteBkt = r.Source.Bucket
|
||||||
pathStyle = r.Source.Path
|
pathStyle = r.Source.Path
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
u, err := url.Parse(remoteEp)
|
u, err := url.Parse(remoteEp)
|
||||||
@ -2310,7 +2309,6 @@ func lookupStyle(s string) miniogo.BucketLookupType {
|
|||||||
lookup = miniogo.BucketLookupDNS
|
lookup = miniogo.BucketLookupDNS
|
||||||
default:
|
default:
|
||||||
lookup = miniogo.BucketLookupAuto
|
lookup = miniogo.BucketLookupAuto
|
||||||
|
|
||||||
}
|
}
|
||||||
return lookup
|
return lookup
|
||||||
}
|
}
|
||||||
|
@ -178,7 +178,7 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
|
|||||||
return errFileCorrupt
|
return errFileCorrupt
|
||||||
}
|
}
|
||||||
|
|
||||||
bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
|
bufp := xioutil.ODirectPoolSmall.Get()
|
||||||
defer xioutil.ODirectPoolSmall.Put(bufp)
|
defer xioutil.ODirectPoolSmall.Put(bufp)
|
||||||
|
|
||||||
for left > 0 {
|
for left > 0 {
|
||||||
|
@ -344,11 +344,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
|||||||
Created: dnsRecords[0].CreationDate,
|
Created: dnsRecords[0].CreationDate,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(bucketsInfo, func(i, j int) bool {
|
sort.Slice(bucketsInfo, func(i, j int) bool {
|
||||||
return bucketsInfo[i].Name < bucketsInfo[j].Name
|
return bucketsInfo[i].Name < bucketsInfo[j].Name
|
||||||
})
|
})
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// Invoke the list buckets.
|
// Invoke the list buckets.
|
||||||
var err error
|
var err error
|
||||||
@ -841,7 +839,6 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
|||||||
}
|
}
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
|
|
||||||
}
|
}
|
||||||
apiErr := ErrBucketAlreadyExists
|
apiErr := ErrBucketAlreadyExists
|
||||||
if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||||
|
@ -188,7 +188,6 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
|
|||||||
if errorResponse.Code != testCase.errorResponse.Code {
|
if errorResponse.Code != testCase.errorResponse.Code {
|
||||||
t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
|
t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test for Anonymous/unsigned http request.
|
// Test for Anonymous/unsigned http request.
|
||||||
@ -290,7 +289,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
|
|||||||
if recV2.Code != testCase.expectedRespStatus {
|
if recV2.Code != testCase.expectedRespStatus {
|
||||||
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
|
t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test for Anonymous/unsigned http request.
|
// Test for Anonymous/unsigned http request.
|
||||||
|
@ -112,7 +112,6 @@ func (r *ReplicationStats) collectWorkerMetrics(ctx context.Context) {
|
|||||||
r.wlock.Lock()
|
r.wlock.Lock()
|
||||||
r.workers.update()
|
r.workers.update()
|
||||||
r.wlock.Unlock()
|
r.wlock.Unlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -146,7 +146,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
|||||||
if errInt == nil {
|
if errInt == nil {
|
||||||
err = nil
|
err = nil
|
||||||
} else {
|
} else {
|
||||||
err = errInt.(error)
|
err, _ = errInt.(error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switch err.(type) {
|
switch err.(type) {
|
||||||
|
@ -45,7 +45,7 @@ func Test_readFromSecret(t *testing.T) {
|
|||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
testCase := testCase
|
testCase := testCase
|
||||||
t.Run("", func(t *testing.T) {
|
t.Run("", func(t *testing.T) {
|
||||||
tmpfile, err := os.CreateTemp("", "testfile")
|
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
@ -157,7 +157,7 @@ MINIO_ROOT_PASSWORD=minio123`,
|
|||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
testCase := testCase
|
testCase := testCase
|
||||||
t.Run("", func(t *testing.T) {
|
t.Run("", func(t *testing.T) {
|
||||||
tmpfile, err := os.CreateTemp("", "testfile")
|
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
@ -858,8 +858,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if compact {
|
if compact {
|
||||||
stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
|
||||||
f.newCache.deleteRecursive(thisHash)
|
f.newCache.deleteRecursive(thisHash)
|
||||||
@ -873,7 +873,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
|
|||||||
}
|
}
|
||||||
stop(total)
|
stop(total)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
// Compact if too many children...
|
// Compact if too many children...
|
||||||
if !into.Compacted {
|
if !into.Compacted {
|
||||||
|
@ -98,7 +98,6 @@ func (dt *dynamicTimeout) logEntry(duration time.Duration) {
|
|||||||
|
|
||||||
// We leak entries while we copy
|
// We leak entries while we copy
|
||||||
if entries == dynamicTimeoutLogSize {
|
if entries == dynamicTimeoutLogSize {
|
||||||
|
|
||||||
// Make copy on stack in order to call adjust()
|
// Make copy on stack in order to call adjust()
|
||||||
logCopy := [dynamicTimeoutLogSize]time.Duration{}
|
logCopy := [dynamicTimeoutLogSize]time.Duration{}
|
||||||
copy(logCopy[:], dt.log[:])
|
copy(logCopy[:], dt.log[:])
|
||||||
|
@ -167,7 +167,6 @@ func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() fl
|
|||||||
const successTimeout = 20 * time.Second
|
const successTimeout = 20 * time.Second
|
||||||
|
|
||||||
for i := 0; i < dynamicTimeoutLogSize; i++ {
|
for i := 0; i < dynamicTimeoutLogSize; i++ {
|
||||||
|
|
||||||
rnd := f()
|
rnd := f()
|
||||||
duration := time.Duration(float64(successTimeout) * rnd)
|
duration := time.Duration(float64(successTimeout) * rnd)
|
||||||
|
|
||||||
|
@ -347,8 +347,8 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
|
|||||||
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
|
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
|
||||||
}
|
}
|
||||||
return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied.
|
return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied.
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm {
|
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm {
|
||||||
return nil // don't rotate on equal keys if seal algorithm is latest
|
return nil // don't rotate on equal keys if seal algorithm is latest
|
||||||
}
|
}
|
||||||
|
@ -362,7 +362,6 @@ func TestGetDecryptedRange(t *testing.T) {
|
|||||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Multipart object tests
|
// Multipart object tests
|
||||||
@ -538,7 +537,6 @@ func TestGetDecryptedRange(t *testing.T) {
|
|||||||
i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
|
i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -213,7 +213,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
|||||||
u.Path = u.Path[1:]
|
u.Path = u.Path[1:]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// Only check if the arg is an ip address and ask for scheme since its absent.
|
// Only check if the arg is an ip address and ask for scheme since its absent.
|
||||||
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
||||||
|
@ -201,7 +201,6 @@ func erasureSelfTest() {
|
|||||||
ok = false
|
ok = false
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -304,7 +304,6 @@ func TestListOnlineDisks(t *testing.T) {
|
|||||||
f.Close()
|
f.Close()
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
||||||
@ -485,7 +484,6 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
|
|||||||
f.Close()
|
f.Close()
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
|
||||||
|
@ -584,7 +584,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
|||||||
readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
|
readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
|
||||||
checksumInfo.Hash, erasure.ShardSize())
|
checksumInfo.Hash, erasure.ShardSize())
|
||||||
prefer[i] = disk.Hostname() == ""
|
prefer[i] = disk.Hostname() == ""
|
||||||
|
|
||||||
}
|
}
|
||||||
writers := make([]io.Writer, len(outDatedDisks))
|
writers := make([]io.Writer, len(outDatedDisks))
|
||||||
for i, disk := range outDatedDisks {
|
for i, disk := range outDatedDisks {
|
||||||
@ -643,9 +642,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
|||||||
if disksToHealCount == 0 {
|
if disksToHealCount == 0 {
|
||||||
return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object)
|
return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID)
|
defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID)
|
||||||
|
@ -1049,7 +1049,6 @@ func readParts(ctx context.Context, disks []StorageAPI, bucket string, partMetaP
|
|||||||
PartNumber: partNumbers[pidx],
|
PartNumber: partNumbers[pidx],
|
||||||
}.Error(),
|
}.Error(),
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return partInfosInQuorum, nil
|
return partInfosInQuorum, nil
|
||||||
}
|
}
|
||||||
|
@ -246,7 +246,6 @@ func TestDeleteObjectsVersioned(t *testing.T) {
|
|||||||
VersionID: objInfo.VersionID,
|
VersionID: objInfo.VersionID,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
names = append(names, ObjectToDelete{
|
names = append(names, ObjectToDelete{
|
||||||
ObjectV: ObjectV{
|
ObjectV: ObjectV{
|
||||||
|
@ -366,7 +366,6 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
|
|||||||
writeErrorResponse(ctx, w,
|
writeErrorResponse(ctx, w,
|
||||||
errorCodes.ToAPIErr(ErrTooManyRequests),
|
errorCodes.ToAPIErr(ErrTooManyRequests),
|
||||||
r.URL)
|
r.URL)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -493,7 +493,6 @@ func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWa
|
|||||||
keyPath: string(event.Kv.Key),
|
keyPath: string(event.Kv.Key),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -278,7 +278,6 @@ func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string,
|
|||||||
iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
|
iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
|
||||||
}
|
}
|
||||||
return u, errNoSuchUser
|
return u, errNoSuchUser
|
||||||
|
|
||||||
}
|
}
|
||||||
u.Credentials.Claims = jwtClaims.Map()
|
u.Credentials.Claims = jwtClaims.Map()
|
||||||
}
|
}
|
||||||
|
@ -845,7 +845,11 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return val.([]string), nil
|
res, ok := val.([]string)
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("unexpected policy type")
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
}
|
}
|
||||||
return getPolicies()
|
return getPolicies()
|
||||||
}
|
}
|
||||||
@ -1218,7 +1222,6 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
|
|||||||
cache.iamGroupPolicyMap.Delete(name)
|
cache.iamGroupPolicyMap.Delete(name)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
|
if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -1620,7 +1623,6 @@ func (store *IAMStoreSys) MergePolicies(policyName string) (string, policy.Polic
|
|||||||
policies = append(policies, policy)
|
policies = append(policies, policy)
|
||||||
toMerge = append(toMerge, p.Policy)
|
toMerge = append(toMerge, p.Policy)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
|
return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
|
||||||
@ -2917,7 +2919,10 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
newCache := val.(*iamCache)
|
newCache, ok := val.(*iamCache)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
cache := store.lock()
|
cache := store.lock()
|
||||||
defer store.unlock()
|
defer store.unlock()
|
||||||
|
@ -2286,7 +2286,6 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
|
|||||||
}
|
}
|
||||||
policies = policySet.ToSlice()
|
policies = policySet.ToSlice()
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defensive code: Do not allow any operation if no policy is found in the session token
|
// Defensive code: Do not allow any operation if no policy is found in the session token
|
||||||
|
@ -26,6 +26,7 @@ import (
|
|||||||
|
|
||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/grid"
|
"github.com/minio/minio/internal/grid"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/minio/internal/pubsub"
|
"github.com/minio/minio/internal/pubsub"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
@ -200,19 +201,19 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
|
|||||||
}
|
}
|
||||||
if len(mergeCh) == 0 {
|
if len(mergeCh) == 0 {
|
||||||
// Flush if nothing is queued
|
// Flush if nothing is queued
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
grid.PutByteBuffer(ev)
|
grid.PutByteBuffer(ev)
|
||||||
case <-emptyEventTicker:
|
case <-emptyEventTicker:
|
||||||
if err := enc.Encode(struct{ Records []event.Event }{}); err != nil {
|
if err := enc.Encode(struct{ Records []event.Event }{}); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-keepAliveTicker:
|
case <-keepAliveTicker:
|
||||||
if _, err := w.Write([]byte(" ")); err != nil {
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -136,7 +136,6 @@ func TestLocalLockerUnlock(t *testing.T) {
|
|||||||
}
|
}
|
||||||
wResources[i] = names
|
wResources[i] = names
|
||||||
wUIDs[i] = uid
|
wUIDs[i] = uid
|
||||||
|
|
||||||
}
|
}
|
||||||
for i := range rResources {
|
for i := range rResources {
|
||||||
name := mustGetUUID()
|
name := mustGetUUID()
|
||||||
|
@ -27,6 +27,7 @@ import (
|
|||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
"github.com/klauspost/compress/s2"
|
"github.com/klauspost/compress/s2"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
"github.com/valyala/bytebufferpool"
|
"github.com/valyala/bytebufferpool"
|
||||||
@ -236,7 +237,7 @@ func (w *metacacheWriter) Reset(out io.Writer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var s2DecPool = sync.Pool{New: func() interface{} {
|
var s2DecPool = bpool.Pool[*s2.Reader]{New: func() *s2.Reader {
|
||||||
// Default alloc block for network transfer.
|
// Default alloc block for network transfer.
|
||||||
return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10))
|
return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10))
|
||||||
}}
|
}}
|
||||||
@ -253,7 +254,7 @@ type metacacheReader struct {
|
|||||||
// newMetacacheReader creates a new cache reader.
|
// newMetacacheReader creates a new cache reader.
|
||||||
// Nothing will be read from the stream yet.
|
// Nothing will be read from the stream yet.
|
||||||
func newMetacacheReader(r io.Reader) *metacacheReader {
|
func newMetacacheReader(r io.Reader) *metacacheReader {
|
||||||
dec := s2DecPool.Get().(*s2.Reader)
|
dec := s2DecPool.Get()
|
||||||
dec.Reset(r)
|
dec.Reset(r)
|
||||||
mr := msgpNewReader(dec)
|
mr := msgpNewReader(dec)
|
||||||
return &metacacheReader{
|
return &metacacheReader{
|
||||||
|
@ -2474,7 +2474,6 @@ func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
|
|||||||
}
|
}
|
||||||
downtimeDuration.Value = float64(dwntime / time.Second)
|
downtimeDuration.Value = float64(dwntime / time.Second)
|
||||||
ml = append(ml, downtimeDuration)
|
ml = append(ml, downtimeDuration)
|
||||||
|
|
||||||
}
|
}
|
||||||
return ml
|
return ml
|
||||||
})
|
})
|
||||||
|
@ -177,7 +177,6 @@ func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metri
|
|||||||
for k, v := range usage.ObjectVersionsHistogram {
|
for k, v := range usage.ObjectVersionsHistogram {
|
||||||
m.Set(usageBucketObjectVersionCountDistribution, float64(v), "range", k, "bucket", bucket)
|
m.Set(usageBucketObjectVersionCountDistribution, float64(v), "range", k, "bucket", bucket)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -1184,7 +1184,6 @@ func (sys *NotificationSys) GetPeerOnlineCount() (nodesOnline, nodesOffline int)
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
nodesOnlineIndex[idx] = client.restClient.HealthCheckFn()
|
nodesOnlineIndex[idx] = client.restClient.HealthCheckFn()
|
||||||
}(idx, client)
|
}(idx, client)
|
||||||
|
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
|
@ -389,7 +389,6 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
||||||
@ -1014,7 +1013,6 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
|
|||||||
t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
|
t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -1166,7 +1164,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Formualting the result data set to be expected from ListObjects call inside the tests,
|
// Formualting the result data set to be expected from ListObjects call inside the tests,
|
||||||
@ -1785,12 +1782,10 @@ func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestEr
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
// Formulating the result data set to be expected from ListObjects call inside the tests,
|
||||||
// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
|
// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
|
||||||
|
|
||||||
resultCases := []ListObjectsInfo{
|
resultCases := []ListObjectsInfo{
|
||||||
{
|
{
|
||||||
Objects: []ObjectInfo{
|
Objects: []ObjectInfo{
|
||||||
|
@ -443,7 +443,6 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expected Results set for asserting ListObjectMultipart test.
|
// Expected Results set for asserting ListObjectMultipart test.
|
||||||
|
@ -1538,7 +1538,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
|
||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
|
||||||
@ -1631,7 +1630,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
srcInfo.metadataOnly && srcOpts.VersionID == "" &&
|
srcInfo.metadataOnly && srcOpts.VersionID == "" &&
|
||||||
!crypto.Requested(r.Header) &&
|
!crypto.Requested(r.Header) &&
|
||||||
!crypto.IsSourceEncrypted(srcInfo.UserDefined) {
|
!crypto.IsSourceEncrypted(srcInfo.UserDefined) {
|
||||||
|
|
||||||
// If x-amz-metadata-directive is not set to REPLACE then we need
|
// If x-amz-metadata-directive is not set to REPLACE then we need
|
||||||
// to error out if source and destination are same.
|
// to error out if source and destination are same.
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL)
|
||||||
@ -2410,7 +2408,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
|
if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
|
||||||
metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var objectEncryptionKey crypto.ObjectKey
|
var objectEncryptionKey crypto.ObjectKey
|
||||||
|
@ -814,7 +814,6 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
|
|||||||
caseNumber++
|
caseNumber++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// HTTP request for testing when `objectLayer` is set to `nil`.
|
// HTTP request for testing when `objectLayer` is set to `nil`.
|
||||||
@ -1567,8 +1566,8 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if testCase.expectedRespStatus == http.StatusOK {
|
if testCase.expectedRespStatus == http.StatusOK {
|
||||||
buffer := new(bytes.Buffer)
|
buffer := new(bytes.Buffer)
|
||||||
// Fetch the object to check whether the content is same as the one uploaded via PutObject.
|
// Fetch the object to check whether the content is same as the one uploaded via PutObject.
|
||||||
@ -3520,7 +3519,6 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
|
|||||||
t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1,
|
t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1,
|
||||||
instanceType, testCase.expectedRespStatus, recV2.Code)
|
instanceType, testCase.expectedRespStatus, recV2.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test for Anonymous/unsigned http request.
|
// Test for Anonymous/unsigned http request.
|
||||||
@ -4198,7 +4196,6 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
|
|||||||
|
|
||||||
// validate the error response.
|
// validate the error response.
|
||||||
if test.expectedErr != noAPIErr {
|
if test.expectedErr != noAPIErr {
|
||||||
|
|
||||||
var errBytes []byte
|
var errBytes []byte
|
||||||
// read the response body.
|
// read the response body.
|
||||||
errBytes, err = io.ReadAll(rec.Result().Body)
|
errBytes, err = io.ReadAll(rec.Result().Body)
|
||||||
|
@ -228,7 +228,6 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
|
|||||||
if objInfo.Size != int64(len(value)) {
|
if objInfo.Size != int64(len(value)) {
|
||||||
t.Errorf("%s: Size mismatch of the GetObject data.", instanceType)
|
t.Errorf("%s: Size mismatch of the GetObject data.", instanceType)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -384,7 +383,6 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
|||||||
|
|
||||||
// check results with Marker.
|
// check results with Marker.
|
||||||
{
|
{
|
||||||
|
|
||||||
result, err = obj.ListObjects(context.Background(), "bucket", "", "newPrefix", "", 3)
|
result, err = obj.ListObjects(context.Background(), "bucket", "", "newPrefix", "", 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s: <ERROR> %s", instanceType, err)
|
t.Fatalf("%s: <ERROR> %s", instanceType, err)
|
||||||
|
@ -25,10 +25,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"syscall"
|
"syscall"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"golang.org/x/sys/unix"
|
"golang.org/x/sys/unix"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -106,15 +106,15 @@ const blockSize = 8 << 10 // 8192
|
|||||||
|
|
||||||
// By default at least 128 entries in single getdents call (1MiB buffer)
|
// By default at least 128 entries in single getdents call (1MiB buffer)
|
||||||
var (
|
var (
|
||||||
direntPool = sync.Pool{
|
direntPool = bpool.Pool[*[]byte]{
|
||||||
New: func() interface{} {
|
New: func() *[]byte {
|
||||||
buf := make([]byte, blockSize*128)
|
buf := make([]byte, blockSize*128)
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
direntNamePool = sync.Pool{
|
direntNamePool = bpool.Pool[*[]byte]{
|
||||||
New: func() interface{} {
|
New: func() *[]byte {
|
||||||
buf := make([]byte, blockSize)
|
buf := make([]byte, blockSize)
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
@ -183,11 +183,10 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
|
|||||||
}
|
}
|
||||||
return osErrToFileErr(err)
|
return osErrToFileErr(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
defer syscall.Close(fd)
|
defer syscall.Close(fd)
|
||||||
|
|
||||||
bufp := direntPool.Get().(*[]byte)
|
bufp := direntPool.Get()
|
||||||
defer direntPool.Put(bufp)
|
defer direntPool.Put(bufp)
|
||||||
buf := *bufp
|
buf := *bufp
|
||||||
|
|
||||||
@ -273,11 +272,11 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
|
|||||||
}
|
}
|
||||||
defer syscall.Close(fd)
|
defer syscall.Close(fd)
|
||||||
|
|
||||||
bufp := direntPool.Get().(*[]byte)
|
bufp := direntPool.Get()
|
||||||
defer direntPool.Put(bufp)
|
defer direntPool.Put(bufp)
|
||||||
buf := *bufp
|
buf := *bufp
|
||||||
|
|
||||||
nameTmp := direntNamePool.Get().(*[]byte)
|
nameTmp := direntNamePool.Get()
|
||||||
defer direntNamePool.Put(nameTmp)
|
defer direntNamePool.Put(nameTmp)
|
||||||
tmp := *nameTmp
|
tmp := *nameTmp
|
||||||
|
|
||||||
|
@ -173,7 +173,6 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
|
|||||||
|
|
||||||
count--
|
count--
|
||||||
entries = append(entries, name)
|
entries = append(entries, name)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return entries, nil
|
return entries, nil
|
||||||
|
@ -131,16 +131,17 @@ func sanitizePolicy(r io.Reader) (io.Reader, error) {
|
|||||||
d := jstream.NewDecoder(r, 0).ObjectAsKVS().MaxDepth(10)
|
d := jstream.NewDecoder(r, 0).ObjectAsKVS().MaxDepth(10)
|
||||||
sset := set.NewStringSet()
|
sset := set.NewStringSet()
|
||||||
for mv := range d.Stream() {
|
for mv := range d.Stream() {
|
||||||
var kvs jstream.KVS
|
|
||||||
if mv.ValueType == jstream.Object {
|
if mv.ValueType == jstream.Object {
|
||||||
// This is a JSON object type (that preserves key order)
|
// This is a JSON object type (that preserves key order)
|
||||||
kvs = mv.Value.(jstream.KVS)
|
kvs, ok := mv.Value.(jstream.KVS)
|
||||||
for _, kv := range kvs {
|
if ok {
|
||||||
if sset.Contains(kv.Key) {
|
for _, kv := range kvs {
|
||||||
// Reject duplicate conditions or expiration.
|
if sset.Contains(kv.Key) {
|
||||||
return nil, fmt.Errorf("input policy has multiple %s, please fix your client code", kv.Key)
|
// Reject duplicate conditions or expiration.
|
||||||
|
return nil, fmt.Errorf("input policy has multiple %s, please fix your client code", kv.Key)
|
||||||
|
}
|
||||||
|
sset.Add(kv.Key)
|
||||||
}
|
}
|
||||||
sset.Add(kv.Key)
|
|
||||||
}
|
}
|
||||||
e.Encode(kvs)
|
e.Encode(kvs)
|
||||||
}
|
}
|
||||||
|
@ -174,7 +174,6 @@ internalAuth:
|
|||||||
if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) != 1 {
|
if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) != 1 {
|
||||||
return nil, errAuthentication
|
return nil, errAuthentication
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
copts := map[string]string{
|
copts := map[string]string{
|
||||||
@ -223,14 +222,11 @@ func processLDAPAuthentication(key ssh.PublicKey, pass []byte, user string) (per
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if key != nil {
|
} else if key != nil {
|
||||||
|
|
||||||
lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(user)
|
lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(user)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if lookupResult == nil {
|
if lookupResult == nil {
|
||||||
|
@ -159,7 +159,6 @@ func TestDoesPresignedV2SignatureMatch(t *testing.T) {
|
|||||||
t.Errorf("(%d) expected to get success, instead got %s", i, niceError(errCode))
|
t.Errorf("(%d) expected to get success, instead got %s", i, niceError(errCode))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -298,7 +298,6 @@ func TestParseSignature(t *testing.T) {
|
|||||||
t.Errorf("Test %d: Expected the result to be \"%s\", but got \"%s\". ", i+1, testCase.expectedSignStr, actualSignStr)
|
t.Errorf("Test %d: Expected the result to be \"%s\", but got \"%s\". ", i+1, testCase.expectedSignStr, actualSignStr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -343,7 +342,6 @@ func TestParseSignedHeaders(t *testing.T) {
|
|||||||
t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedSignedHeaders, actualSignedHeaders)
|
t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedSignedHeaders, actualSignedHeaders)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -514,7 +512,6 @@ func TestParseSignV4(t *testing.T) {
|
|||||||
t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedAuthField, parsedAuthField.SignedHeaders)
|
t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedAuthField, parsedAuthField.SignedHeaders)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -880,6 +877,5 @@ func TestParsePreSignV4(t *testing.T) {
|
|||||||
t.Errorf("Test %d: Expected date to be %v, but got %v", i+1, testCase.expectedPreSignValues.Date.UTC().Format(iso8601Format), parsedPreSign.Date.UTC().Format(iso8601Format))
|
t.Errorf("Test %d: Expected date to be %v, but got %v", i+1, testCase.expectedPreSignValues.Date.UTC().Format(iso8601Format), parsedPreSign.Date.UTC().Format(iso8601Format))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1037,7 +1037,6 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
|
|||||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
|
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
|
||||||
return wrapSRErr(err)
|
return wrapSRErr(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
// no replication rule for this peer or target ARN missing in bucket targets
|
// no replication rule for this peer or target ARN missing in bucket targets
|
||||||
if targetARN == "" {
|
if targetARN == "" {
|
||||||
@ -1406,7 +1405,6 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
|
|||||||
if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil {
|
if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil {
|
||||||
return wrapSRErr(err)
|
return wrapSRErr(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@ -1430,8 +1428,8 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
|
|||||||
userType := IAMUserType(mapping.UserType)
|
userType := IAMUserType(mapping.UserType)
|
||||||
isGroup := mapping.IsGroup
|
isGroup := mapping.IsGroup
|
||||||
entityName := mapping.UserOrGroup
|
entityName := mapping.UserOrGroup
|
||||||
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
|
|
||||||
|
|
||||||
|
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser {
|
||||||
// Validate that the user or group exists in LDAP and use the normalized
|
// Validate that the user or group exists in LDAP and use the normalized
|
||||||
// form of the entityName (which will be an LDAP DN).
|
// form of the entityName (which will be an LDAP DN).
|
||||||
var err error
|
var err error
|
||||||
@ -3062,7 +3060,6 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
|
|||||||
sum.ReplicatedGroupPolicyMappings++
|
sum.ReplicatedGroupPolicyMappings++
|
||||||
info.StatsSummary[ps.DeploymentID] = sum
|
info.StatsSummary[ps.DeploymentID] = sum
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -29,11 +29,11 @@ import (
|
|||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/cachevalue"
|
"github.com/minio/minio/internal/cachevalue"
|
||||||
"github.com/minio/minio/internal/grid"
|
"github.com/minio/minio/internal/grid"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
@ -508,13 +508,13 @@ func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcP
|
|||||||
}
|
}
|
||||||
|
|
||||||
// where we keep old *Readers
|
// where we keep old *Readers
|
||||||
var readMsgpReaderPool = sync.Pool{New: func() interface{} { return &msgp.Reader{} }}
|
var readMsgpReaderPool = bpool.Pool[*msgp.Reader]{New: func() *msgp.Reader { return &msgp.Reader{} }}
|
||||||
|
|
||||||
// mspNewReader returns a *Reader that reads from the provided reader.
|
// mspNewReader returns a *Reader that reads from the provided reader.
|
||||||
// The reader will be buffered.
|
// The reader will be buffered.
|
||||||
// Return with readMsgpReaderPoolPut when done.
|
// Return with readMsgpReaderPoolPut when done.
|
||||||
func msgpNewReader(r io.Reader) *msgp.Reader {
|
func msgpNewReader(r io.Reader) *msgp.Reader {
|
||||||
p := readMsgpReaderPool.Get().(*msgp.Reader)
|
p := readMsgpReaderPool.Get()
|
||||||
if p.R == nil {
|
if p.R == nil {
|
||||||
p.R = xbufio.NewReaderSize(r, 32<<10)
|
p.R = xbufio.NewReaderSize(r, 32<<10)
|
||||||
} else {
|
} else {
|
||||||
|
@ -34,6 +34,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/grid"
|
"github.com/minio/minio/internal/grid"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
|
||||||
@ -831,7 +832,7 @@ func keepHTTPReqResponseAlive(w http.ResponseWriter, r *http.Request) (resp func
|
|||||||
// Response not ready, write a filler byte.
|
// Response not ready, write a filler byte.
|
||||||
write([]byte{32})
|
write([]byte{32})
|
||||||
if canWrite {
|
if canWrite {
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case err := <-doneCh:
|
case err := <-doneCh:
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -905,7 +906,7 @@ func keepHTTPResponseAlive(w http.ResponseWriter) func(error) {
|
|||||||
// Response not ready, write a filler byte.
|
// Response not ready, write a filler byte.
|
||||||
write([]byte{32})
|
write([]byte{32})
|
||||||
if canWrite {
|
if canWrite {
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case err := <-doneCh:
|
case err := <-doneCh:
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1025,7 +1026,7 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
|
|||||||
// Response not ready, write a filler byte.
|
// Response not ready, write a filler byte.
|
||||||
write([]byte{32})
|
write([]byte{32})
|
||||||
if canWrite {
|
if canWrite {
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
case err := <-doneCh:
|
case err := <-doneCh:
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1043,7 +1044,7 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
|
|||||||
write(tmp[:])
|
write(tmp[:])
|
||||||
write(block)
|
write(block)
|
||||||
if canWrite {
|
if canWrite {
|
||||||
w.(http.Flusher).Flush()
|
xhttp.Flush(w)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1051,29 +1052,23 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
|
|||||||
return &h
|
return &h
|
||||||
}
|
}
|
||||||
|
|
||||||
var poolBuf8k = sync.Pool{
|
var poolBuf8k = bpool.Pool[*[]byte]{
|
||||||
New: func() interface{} {
|
New: func() *[]byte {
|
||||||
b := make([]byte, 8192)
|
b := make([]byte, 8192)
|
||||||
return &b
|
return &b
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var poolBuf128k = sync.Pool{
|
|
||||||
New: func() interface{} {
|
|
||||||
b := make([]byte, 128<<10)
|
|
||||||
return b
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForHTTPStream will wait for responses where
|
// waitForHTTPStream will wait for responses where
|
||||||
// streamHTTPResponse has been used.
|
// streamHTTPResponse has been used.
|
||||||
// The returned reader contains the payload and must be closed if no error is returned.
|
// The returned reader contains the payload and must be closed if no error is returned.
|
||||||
func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
|
||||||
var tmp [1]byte
|
var tmp [1]byte
|
||||||
// 8K copy buffer, reused for less allocs...
|
// 8K copy buffer, reused for less allocs...
|
||||||
bufp := poolBuf8k.Get().(*[]byte)
|
bufp := poolBuf8k.Get()
|
||||||
buf := *bufp
|
buf := *bufp
|
||||||
defer poolBuf8k.Put(bufp)
|
defer poolBuf8k.Put(bufp)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_, err := io.ReadFull(respBody, tmp[:])
|
_, err := io.ReadFull(respBody, tmp[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1438,7 +1433,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}(endpoint)
|
}(endpoint)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -794,7 +794,6 @@ func assembleStreamingChunks(req *http.Request, body io.ReadSeeker, chunkSize in
|
|||||||
if n <= 0 {
|
if n <= 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
req.Body = io.NopCloser(bytes.NewReader(stream))
|
req.Body = io.NopCloser(bytes.NewReader(stream))
|
||||||
return req, nil
|
return req, nil
|
||||||
|
14
cmd/untar.go
14
cmd/untar.go
@ -36,6 +36,7 @@ import (
|
|||||||
"github.com/klauspost/compress/s2"
|
"github.com/klauspost/compress/s2"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
gzip "github.com/klauspost/pgzip"
|
gzip "github.com/klauspost/pgzip"
|
||||||
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/pierrec/lz4/v4"
|
"github.com/pierrec/lz4/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -182,7 +183,6 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
|
|||||||
|
|
||||||
header, err := tarReader.Next()
|
header, err := tarReader.Next()
|
||||||
switch {
|
switch {
|
||||||
|
|
||||||
// if no more files are found return
|
// if no more files are found return
|
||||||
case err == io.EOF:
|
case err == io.EOF:
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
@ -226,13 +226,10 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
|
|||||||
|
|
||||||
// Do small files async
|
// Do small files async
|
||||||
n++
|
n++
|
||||||
if header.Size <= smallFileThreshold {
|
if header.Size <= xioutil.MediumBlock {
|
||||||
asyncWriters <- struct{}{}
|
asyncWriters <- struct{}{}
|
||||||
b := poolBuf128k.Get().([]byte)
|
bufp := xioutil.ODirectPoolMedium.Get()
|
||||||
if cap(b) < int(header.Size) {
|
b := (*bufp)[:header.Size]
|
||||||
b = make([]byte, smallFileThreshold)
|
|
||||||
}
|
|
||||||
b = b[:header.Size]
|
|
||||||
if _, err := io.ReadFull(tarReader, b); err != nil {
|
if _, err := io.ReadFull(tarReader, b); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -243,8 +240,7 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
|
|||||||
rc.Close()
|
rc.Close()
|
||||||
<-asyncWriters
|
<-asyncWriters
|
||||||
wg.Done()
|
wg.Done()
|
||||||
//nolint:staticcheck // SA6002 we are fine with the tiny alloc
|
xioutil.ODirectPoolMedium.Put(bufp)
|
||||||
poolBuf128k.Put(b)
|
|
||||||
}()
|
}()
|
||||||
if err := putObject(&rc, fi, name); err != nil {
|
if err := putObject(&rc, fi, name); err != nil {
|
||||||
if o.ignoreErrs {
|
if o.ignoreErrs {
|
||||||
|
@ -210,7 +210,7 @@ func TestIsKubernetes(t *testing.T) {
|
|||||||
// Tests if the environment we are running is Helm chart.
|
// Tests if the environment we are running is Helm chart.
|
||||||
func TestGetHelmVersion(t *testing.T) {
|
func TestGetHelmVersion(t *testing.T) {
|
||||||
createTempFile := func(content string) string {
|
createTempFile := func(content string) string {
|
||||||
tmpfile, err := os.CreateTemp("", "helm-testfile-")
|
tmpfile, err := os.CreateTemp(t.TempDir(), "helm-testfile-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Unable to create temporary file. %s", err)
|
t.Fatalf("Unable to create temporary file. %s", err)
|
||||||
}
|
}
|
||||||
|
@ -26,12 +26,12 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/cespare/xxhash/v2"
|
"github.com/cespare/xxhash/v2"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
"github.com/minio/minio/internal/bucket/replication"
|
"github.com/minio/minio/internal/bucket/replication"
|
||||||
"github.com/minio/minio/internal/config/storageclass"
|
"github.com/minio/minio/internal/config/storageclass"
|
||||||
@ -703,18 +703,17 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string, allParts bool) (FileInfo
|
|||||||
const metaDataReadDefault = 4 << 10
|
const metaDataReadDefault = 4 << 10
|
||||||
|
|
||||||
// Return used metadata byte slices here.
|
// Return used metadata byte slices here.
|
||||||
var metaDataPool = sync.Pool{New: func() interface{} { return make([]byte, 0, metaDataReadDefault) }}
|
var metaDataPool = bpool.Pool[[]byte]{New: func() []byte { return make([]byte, 0, metaDataReadDefault) }}
|
||||||
|
|
||||||
// metaDataPoolGet will return a byte slice with capacity at least metaDataReadDefault.
|
// metaDataPoolGet will return a byte slice with capacity at least metaDataReadDefault.
|
||||||
// It will be length 0.
|
// It will be length 0.
|
||||||
func metaDataPoolGet() []byte {
|
func metaDataPoolGet() []byte {
|
||||||
return metaDataPool.Get().([]byte)[:0]
|
return metaDataPool.Get()[:0]
|
||||||
}
|
}
|
||||||
|
|
||||||
// metaDataPoolPut will put an unused small buffer back into the pool.
|
// metaDataPoolPut will put an unused small buffer back into the pool.
|
||||||
func metaDataPoolPut(buf []byte) {
|
func metaDataPoolPut(buf []byte) {
|
||||||
if cap(buf) >= metaDataReadDefault && cap(buf) < metaDataReadDefault*4 {
|
if cap(buf) >= metaDataReadDefault && cap(buf) < metaDataReadDefault*4 {
|
||||||
//nolint:staticcheck // SA6002 we are fine with the tiny alloc
|
|
||||||
metaDataPool.Put(buf)
|
metaDataPool.Put(buf)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1982,7 +1981,6 @@ func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions
|
|||||||
if !latest.header.FreeVersion() {
|
if !latest.header.FreeVersion() {
|
||||||
nVersions++
|
nVersions++
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// Find latest.
|
// Find latest.
|
||||||
var latestCount int
|
var latestCount int
|
||||||
|
@ -225,7 +225,6 @@ func compareXLMetaV1(t *testing.T, unMarshalXLMeta, jsoniterXLMeta xlMetaV1Objec
|
|||||||
if val != jsoniterVal {
|
if val != jsoniterVal {
|
||||||
t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, jsoniterVal)
|
t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, jsoniterVal)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2166,10 +2166,10 @@ func (s *xlStorage) writeAllDirect(ctx context.Context, filePath string, fileSiz
|
|||||||
var bufp *[]byte
|
var bufp *[]byte
|
||||||
switch {
|
switch {
|
||||||
case fileSize <= xioutil.SmallBlock:
|
case fileSize <= xioutil.SmallBlock:
|
||||||
bufp = xioutil.ODirectPoolSmall.Get().(*[]byte)
|
bufp = xioutil.ODirectPoolSmall.Get()
|
||||||
defer xioutil.ODirectPoolSmall.Put(bufp)
|
defer xioutil.ODirectPoolSmall.Put(bufp)
|
||||||
default:
|
default:
|
||||||
bufp = xioutil.ODirectPoolLarge.Get().(*[]byte)
|
bufp = xioutil.ODirectPoolLarge.Get()
|
||||||
defer xioutil.ODirectPoolLarge.Put(bufp)
|
defer xioutil.ODirectPoolLarge.Put(bufp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -369,8 +369,8 @@ FLAGS:
|
|||||||
defer f.Close()
|
defer f.Close()
|
||||||
r = f
|
r = f
|
||||||
}
|
}
|
||||||
if strings.HasSuffix(file, ".zip") {
|
if ra, ok := r.(io.ReaderAt); ok && strings.HasSuffix(file, ".zip") {
|
||||||
zr, err := zip.NewReader(r.(io.ReaderAt), sz)
|
zr, err := zip.NewReader(ra, sz)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
45
internal/bpool/pool.go
Normal file
45
internal/bpool/pool.go
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package bpool
|
||||||
|
|
||||||
|
import "sync"
|
||||||
|
|
||||||
|
// Pool is a single type sync.Pool with a few extra properties:
|
||||||
|
// If New is not set Get may return the zero value of T.
|
||||||
|
type Pool[T any] struct {
|
||||||
|
New func() T
|
||||||
|
p sync.Pool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get will retuen a new T
|
||||||
|
func (p *Pool[T]) Get() T {
|
||||||
|
v, ok := p.p.Get().(T)
|
||||||
|
if ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
if p.New == nil {
|
||||||
|
var t T
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return p.New()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put a used T.
|
||||||
|
func (p *Pool[T]) Put(t T) {
|
||||||
|
p.p.Put(t)
|
||||||
|
}
|
@ -127,7 +127,6 @@ func (m *Monitor) getReport(selectBucket SelectionFunction) *BucketBandwidthRepo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.tlock.RUnlock()
|
m.tlock.RUnlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
@ -64,7 +64,6 @@ func (r *MonitoredReader) Read(buf []byte) (n int, err error) {
|
|||||||
r.opts.HeaderSize = 0
|
r.opts.HeaderSize = 0
|
||||||
need = int(math.Min(float64(b-hdr), float64(need))) // use remaining tokens towards payload
|
need = int(math.Min(float64(b-hdr), float64(need))) // use remaining tokens towards payload
|
||||||
tokens = need + hdr
|
tokens = need + hdr
|
||||||
|
|
||||||
} else { // part of header can be accommodated
|
} else { // part of header can be accommodated
|
||||||
r.opts.HeaderSize -= b - 1
|
r.opts.HeaderSize -= b - 1
|
||||||
need = 1 // to ensure we read at least one byte for every Read
|
need = 1 // to ensure we read at least one byte for every Read
|
||||||
|
@ -207,7 +207,6 @@ func (lc Lifecycle) HasActiveRules(prefix string) bool {
|
|||||||
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
|
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
@ -365,7 +365,6 @@ func TestHasActiveRules(t *testing.T) {
|
|||||||
t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got)
|
t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -67,6 +67,5 @@ func TestMetadataReplicate(t *testing.T) {
|
|||||||
t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedResult, got)
|
t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedResult, got)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,10 +22,11 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func createTempFile(prefix, content string) (tempFile string, err error) {
|
func createTempFile(t testing.TB, prefix, content string) (tempFile string, err error) {
|
||||||
|
t.Helper()
|
||||||
var tmpfile *os.File
|
var tmpfile *os.File
|
||||||
|
|
||||||
if tmpfile, err = os.CreateTemp("", prefix); err != nil {
|
if tmpfile, err = os.CreateTemp(t.TempDir(), prefix); err != nil {
|
||||||
return tempFile, err
|
return tempFile, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -42,14 +43,13 @@ func createTempFile(prefix, content string) (tempFile string, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParsePublicCertFile(t *testing.T) {
|
func TestParsePublicCertFile(t *testing.T) {
|
||||||
tempFile1, err := createTempFile("public-cert-file", "")
|
tempFile1, err := createTempFile(t, "public-cert-file", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Unable to create temporary file. %v", err)
|
t.Fatalf("Unable to create temporary file. %v", err)
|
||||||
}
|
}
|
||||||
defer os.Remove(tempFile1)
|
defer os.Remove(tempFile1)
|
||||||
|
|
||||||
tempFile2, err := createTempFile("public-cert-file",
|
tempFile2, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
|
||||||
`-----BEGIN CERTIFICATE-----
|
|
||||||
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
||||||
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
||||||
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
||||||
@ -70,8 +70,7 @@ M9ofSEt/bdRD
|
|||||||
}
|
}
|
||||||
defer os.Remove(tempFile2)
|
defer os.Remove(tempFile2)
|
||||||
|
|
||||||
tempFile3, err := createTempFile("public-cert-file",
|
tempFile3, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
|
||||||
`-----BEGIN CERTIFICATE-----
|
|
||||||
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
||||||
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
||||||
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
||||||
@ -92,8 +91,7 @@ M9ofSEt/bdRD
|
|||||||
}
|
}
|
||||||
defer os.Remove(tempFile3)
|
defer os.Remove(tempFile3)
|
||||||
|
|
||||||
tempFile4, err := createTempFile("public-cert-file",
|
tempFile4, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
|
||||||
`-----BEGIN CERTIFICATE-----
|
|
||||||
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
||||||
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
||||||
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
||||||
@ -114,8 +112,7 @@ M9ofSEt/bdRD
|
|||||||
}
|
}
|
||||||
defer os.Remove(tempFile4)
|
defer os.Remove(tempFile4)
|
||||||
|
|
||||||
tempFile5, err := createTempFile("public-cert-file",
|
tempFile5, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE-----
|
||||||
`-----BEGIN CERTIFICATE-----
|
|
||||||
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
|
||||||
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
|
||||||
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
|
||||||
@ -184,11 +181,11 @@ func TestLoadX509KeyPair(t *testing.T) {
|
|||||||
os.Unsetenv(EnvCertPassword)
|
os.Unsetenv(EnvCertPassword)
|
||||||
})
|
})
|
||||||
for i, testCase := range loadX509KeyPairTests {
|
for i, testCase := range loadX509KeyPairTests {
|
||||||
privateKey, err := createTempFile("private.key", testCase.privateKey)
|
privateKey, err := createTempFile(t, "private.key", testCase.privateKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
|
t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
|
||||||
}
|
}
|
||||||
certificate, err := createTempFile("public.crt", testCase.certificate)
|
certificate, err := createTempFile(t, "public.crt", testCase.certificate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Remove(privateKey)
|
os.Remove(privateKey)
|
||||||
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
|
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
|
||||||
|
@ -143,7 +143,6 @@ func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) {
|
|||||||
|
|
||||||
srvRecord.Key = msgUnPath(srvRecord.Key)
|
srvRecord.Key = msgUnPath(srvRecord.Key)
|
||||||
srvRecords = append(srvRecords, srvRecord)
|
srvRecords = append(srvRecords, srvRecord)
|
||||||
|
|
||||||
}
|
}
|
||||||
sort.Slice(srvRecords, func(i int, j int) bool {
|
sort.Slice(srvRecords, func(i int, j int) bool {
|
||||||
return srvRecords[i].Key < srvRecords[j].Key
|
return srvRecords[i].Key < srvRecords[j].Key
|
||||||
|
@ -534,7 +534,6 @@ func (r *Config) GetSettings() madmin.OpenIDSettings {
|
|||||||
HashedClientSecret: hashedSecret,
|
HashedClientSecret: hashedSecret,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res
|
return res
|
||||||
|
@ -125,7 +125,6 @@ func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, t
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
targets = append(targets, t)
|
targets = append(targets, t)
|
||||||
|
|
||||||
}
|
}
|
||||||
case config.NotifyKafkaSubSys:
|
case config.NotifyKafkaSubSys:
|
||||||
kafkaTargets, err := GetNotifyKafka(cfg[config.NotifyKafkaSubSys])
|
kafkaTargets, err := GetNotifyKafka(cfg[config.NotifyKafkaSubSys])
|
||||||
@ -142,7 +141,6 @@ func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, t
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
targets = append(targets, t)
|
targets = append(targets, t)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
case config.NotifyMQTTSubSys:
|
case config.NotifyMQTTSubSys:
|
||||||
|
@ -127,7 +127,6 @@ func LookupConfig(kvs config.KVS, transport http.RoundTripper) (cfg Config, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return cfg, err
|
return cfg, err
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.License = strings.TrimSpace(env.Get(config.EnvMinIOSubnetLicense, kvs.Get(config.License)))
|
cfg.License = strings.TrimSpace(env.Get(config.EnvMinIOSubnetLicense, kvs.Get(config.License)))
|
||||||
@ -142,9 +141,11 @@ func LookupConfig(kvs config.KVS, transport http.RoundTripper) (cfg Config, err
|
|||||||
|
|
||||||
// Make sure to clone the transport before editing the ProxyURL
|
// Make sure to clone the transport before editing the ProxyURL
|
||||||
if proxyURL != nil {
|
if proxyURL != nil {
|
||||||
ctransport := transport.(*http.Transport).Clone()
|
if tr, ok := transport.(*http.Transport); ok {
|
||||||
ctransport.Proxy = http.ProxyURL((*url.URL)(proxyURL))
|
ctransport := tr.Clone()
|
||||||
cfg.transport = ctransport
|
ctransport.Proxy = http.ProxyURL((*url.URL)(proxyURL))
|
||||||
|
cfg.transport = ctransport
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
cfg.transport = transport
|
cfg.transport = transport
|
||||||
}
|
}
|
||||||
|
@ -103,7 +103,7 @@ func TestReadDriveStats(t *testing.T) {
|
|||||||
for _, testCase := range testCases {
|
for _, testCase := range testCases {
|
||||||
testCase := testCase
|
testCase := testCase
|
||||||
t.Run("", func(t *testing.T) {
|
t.Run("", func(t *testing.T) {
|
||||||
tmpfile, err := os.CreateTemp("", "testfile")
|
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
|
@ -85,7 +85,6 @@ var _ = uint64(1 << objectSingleTypesEnd)
|
|||||||
// Expand - returns expanded values of abbreviated event type.
|
// Expand - returns expanded values of abbreviated event type.
|
||||||
func (name Name) Expand() []Name {
|
func (name Name) Expand() []Name {
|
||||||
switch name {
|
switch name {
|
||||||
|
|
||||||
case ObjectAccessedAll:
|
case ObjectAccessedAll:
|
||||||
return []Name{
|
return []Name{
|
||||||
ObjectAccessedGet, ObjectAccessedHead,
|
ObjectAccessedGet, ObjectAccessedHead,
|
||||||
|
@ -463,8 +463,7 @@ func (c *esClientV7) createIndex(args ElasticsearchArgs) error {
|
|||||||
indices, ok := v["indices"].([]interface{})
|
indices, ok := v["indices"].([]interface{})
|
||||||
if ok {
|
if ok {
|
||||||
for _, index := range indices {
|
for _, index := range indices {
|
||||||
name := index.(map[string]interface{})["name"]
|
if name, ok := index.(map[string]interface{}); ok && name["name"] == args.Index {
|
||||||
if name == args.Index {
|
|
||||||
found = true
|
found = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -1748,20 +1748,20 @@ func (c *Connection) debugMsg(d debugMsg, args ...any) {
|
|||||||
case debugSetConnPingDuration:
|
case debugSetConnPingDuration:
|
||||||
c.connMu.Lock()
|
c.connMu.Lock()
|
||||||
defer c.connMu.Unlock()
|
defer c.connMu.Unlock()
|
||||||
c.connPingInterval = args[0].(time.Duration)
|
c.connPingInterval, _ = args[0].(time.Duration)
|
||||||
if c.connPingInterval < time.Second {
|
if c.connPingInterval < time.Second {
|
||||||
panic("CONN ping interval too low")
|
panic("CONN ping interval too low")
|
||||||
}
|
}
|
||||||
case debugSetClientPingDuration:
|
case debugSetClientPingDuration:
|
||||||
c.connMu.Lock()
|
c.connMu.Lock()
|
||||||
defer c.connMu.Unlock()
|
defer c.connMu.Unlock()
|
||||||
c.clientPingInterval = args[0].(time.Duration)
|
c.clientPingInterval, _ = args[0].(time.Duration)
|
||||||
case debugAddToDeadline:
|
case debugAddToDeadline:
|
||||||
c.addDeadline = args[0].(time.Duration)
|
c.addDeadline, _ = args[0].(time.Duration)
|
||||||
case debugIsOutgoingClosed:
|
case debugIsOutgoingClosed:
|
||||||
// params: muxID uint64, isClosed func(bool)
|
// params: muxID uint64, isClosed func(bool)
|
||||||
muxID := args[0].(uint64)
|
muxID, _ := args[0].(uint64)
|
||||||
resp := args[1].(func(b bool))
|
resp, _ := args[1].(func(b bool))
|
||||||
mid, ok := c.outgoing.Load(muxID)
|
mid, ok := c.outgoing.Load(muxID)
|
||||||
if !ok || mid == nil {
|
if !ok || mid == nil {
|
||||||
resp(true)
|
resp(true)
|
||||||
@ -1772,7 +1772,8 @@ func (c *Connection) debugMsg(d debugMsg, args ...any) {
|
|||||||
mid.respMu.Unlock()
|
mid.respMu.Unlock()
|
||||||
case debugBlockInboundMessages:
|
case debugBlockInboundMessages:
|
||||||
c.connMu.Lock()
|
c.connMu.Lock()
|
||||||
block := (<-chan struct{})(args[0].(chan struct{}))
|
a, _ := args[0].(chan struct{})
|
||||||
|
block := (<-chan struct{})(a)
|
||||||
c.blockMessages.Store(&block)
|
c.blockMessages.Store(&block)
|
||||||
c.connMu.Unlock()
|
c.connMu.Unlock()
|
||||||
}
|
}
|
||||||
|
@ -28,11 +28,11 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gobwas/ws"
|
"github.com/gobwas/ws"
|
||||||
"github.com/gobwas/ws/wsutil"
|
"github.com/gobwas/ws/wsutil"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrDisconnected is returned when the connection to the remote has been lost during the call.
|
// ErrDisconnected is returned when the connection to the remote has been lost during the call.
|
||||||
@ -68,15 +68,15 @@ const (
|
|||||||
defaultSingleRequestTimeout = time.Minute
|
defaultSingleRequestTimeout = time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
var internalByteBuffer = sync.Pool{
|
var internalByteBuffer = bpool.Pool[*[]byte]{
|
||||||
New: func() any {
|
New: func() *[]byte {
|
||||||
m := make([]byte, 0, defaultBufferSize)
|
m := make([]byte, 0, defaultBufferSize)
|
||||||
return &m
|
return &m
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var internal32KByteBuffer = sync.Pool{
|
var internal32KByteBuffer = bpool.Pool[*[]byte]{
|
||||||
New: func() any {
|
New: func() *[]byte {
|
||||||
m := make([]byte, 0, biggerBufMin)
|
m := make([]byte, 0, biggerBufMin)
|
||||||
return &m
|
return &m
|
||||||
},
|
},
|
||||||
@ -87,7 +87,7 @@ var internal32KByteBuffer = sync.Pool{
|
|||||||
// When replacing PutByteBuffer should also be replaced
|
// When replacing PutByteBuffer should also be replaced
|
||||||
// There is no minimum size.
|
// There is no minimum size.
|
||||||
var GetByteBuffer = func() []byte {
|
var GetByteBuffer = func() []byte {
|
||||||
b := *internalByteBuffer.Get().(*[]byte)
|
b := *internalByteBuffer.Get()
|
||||||
return b[:0]
|
return b[:0]
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -101,7 +101,7 @@ func GetByteBufferCap(wantSz int) []byte {
|
|||||||
PutByteBuffer(b)
|
PutByteBuffer(b)
|
||||||
}
|
}
|
||||||
if wantSz <= maxBufferSize {
|
if wantSz <= maxBufferSize {
|
||||||
b := *internal32KByteBuffer.Get().(*[]byte)
|
b := *internal32KByteBuffer.Get()
|
||||||
if cap(b) >= wantSz {
|
if cap(b) >= wantSz {
|
||||||
return b[:0]
|
return b[:0]
|
||||||
}
|
}
|
||||||
|
@ -23,8 +23,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/hash/sha256"
|
"github.com/minio/minio/internal/hash/sha256"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
@ -431,12 +431,12 @@ func recycleFunc[RT RoundTripper](newRT func() RT) (newFn func() RT, recycle fun
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pool := sync.Pool{
|
pool := bpool.Pool[RT]{
|
||||||
New: func() interface{} {
|
New: func() RT {
|
||||||
return newRT()
|
return newRT()
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return func() RT { return pool.Get().(RT) },
|
return pool.Get,
|
||||||
func(r RT) {
|
func(r RT) {
|
||||||
if r != rZero {
|
if r != rZero {
|
||||||
//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE!
|
//nolint:staticcheck // SA6002 IT IS A GENERIC VALUE!
|
||||||
@ -632,8 +632,8 @@ type StreamTypeHandler[Payload, Req, Resp RoundTripper] struct {
|
|||||||
// Will be 0 if newReq is nil.
|
// Will be 0 if newReq is nil.
|
||||||
InCapacity int
|
InCapacity int
|
||||||
|
|
||||||
reqPool sync.Pool
|
reqPool bpool.Pool[Req]
|
||||||
respPool sync.Pool
|
respPool bpool.Pool[Resp]
|
||||||
id HandlerID
|
id HandlerID
|
||||||
newPayload func() Payload
|
newPayload func() Payload
|
||||||
nilReq Req
|
nilReq Req
|
||||||
@ -653,13 +653,13 @@ func NewStream[Payload, Req, Resp RoundTripper](h HandlerID, newPayload func() P
|
|||||||
|
|
||||||
s := newStreamHandler[Payload, Req, Resp](h)
|
s := newStreamHandler[Payload, Req, Resp](h)
|
||||||
if newReq != nil {
|
if newReq != nil {
|
||||||
s.reqPool.New = func() interface{} {
|
s.reqPool.New = func() Req {
|
||||||
return newReq()
|
return newReq()
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
s.InCapacity = 0
|
s.InCapacity = 0
|
||||||
}
|
}
|
||||||
s.respPool.New = func() interface{} {
|
s.respPool.New = func() Resp {
|
||||||
return newResp()
|
return newResp()
|
||||||
}
|
}
|
||||||
s.newPayload = newPayload
|
s.newPayload = newPayload
|
||||||
@ -682,7 +682,7 @@ func (h *StreamTypeHandler[Payload, Req, Resp]) NewPayload() Payload {
|
|||||||
// NewRequest creates a new request.
|
// NewRequest creates a new request.
|
||||||
// The struct may be reused, so caller should clear any fields.
|
// The struct may be reused, so caller should clear any fields.
|
||||||
func (h *StreamTypeHandler[Payload, Req, Resp]) NewRequest() Req {
|
func (h *StreamTypeHandler[Payload, Req, Resp]) NewRequest() Req {
|
||||||
return h.reqPool.Get().(Req)
|
return h.reqPool.Get()
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutRequest will accept a request for reuse.
|
// PutRequest will accept a request for reuse.
|
||||||
@ -706,7 +706,7 @@ func (h *StreamTypeHandler[Payload, Req, Resp]) PutResponse(r Resp) {
|
|||||||
// NewResponse creates a new response.
|
// NewResponse creates a new response.
|
||||||
// Handlers can use this to create a reusable response.
|
// Handlers can use this to create a reusable response.
|
||||||
func (h *StreamTypeHandler[Payload, Req, Resp]) NewResponse() Resp {
|
func (h *StreamTypeHandler[Payload, Req, Resp]) NewResponse() Resp {
|
||||||
return h.respPool.Get().(Resp)
|
return h.respPool.Get()
|
||||||
}
|
}
|
||||||
|
|
||||||
func newStreamHandler[Payload, Req, Resp RoundTripper](h HandlerID) *StreamTypeHandler[Payload, Req, Resp] {
|
func newStreamHandler[Payload, Req, Resp RoundTripper](h HandlerID) *StreamTypeHandler[Payload, Req, Resp] {
|
||||||
|
@ -388,6 +388,5 @@ func (m *muxServer) close() {
|
|||||||
if m.outBlock != nil {
|
if m.outBlock != nil {
|
||||||
xioutil.SafeClose(m.outBlock)
|
xioutil.SafeClose(m.outBlock)
|
||||||
m.outBlock = nil
|
m.outBlock = nil
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -27,6 +27,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/tinylib/msgp/msgp"
|
"github.com/tinylib/msgp/msgp"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -53,7 +54,7 @@ func (m *MSS) Get(key string) string {
|
|||||||
// Set a key, value pair.
|
// Set a key, value pair.
|
||||||
func (m *MSS) Set(key, value string) {
|
func (m *MSS) Set(key, value string) {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
*m = mssPool.Get().(map[string]string)
|
*m = mssPool.Get()
|
||||||
}
|
}
|
||||||
(*m)[key] = value
|
(*m)[key] = value
|
||||||
}
|
}
|
||||||
@ -130,7 +131,7 @@ func (m *MSS) Msgsize() int {
|
|||||||
|
|
||||||
// NewMSS returns a new MSS.
|
// NewMSS returns a new MSS.
|
||||||
func NewMSS() *MSS {
|
func NewMSS() *MSS {
|
||||||
m := MSS(mssPool.Get().(map[string]string))
|
m := MSS(mssPool.Get())
|
||||||
for k := range m {
|
for k := range m {
|
||||||
delete(m, k)
|
delete(m, k)
|
||||||
}
|
}
|
||||||
@ -143,8 +144,8 @@ func NewMSSWith(m map[string]string) *MSS {
|
|||||||
return &m2
|
return &m2
|
||||||
}
|
}
|
||||||
|
|
||||||
var mssPool = sync.Pool{
|
var mssPool = bpool.Pool[map[string]string]{
|
||||||
New: func() interface{} {
|
New: func() map[string]string {
|
||||||
return make(map[string]string, 5)
|
return make(map[string]string, 5)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -152,7 +153,7 @@ var mssPool = sync.Pool{
|
|||||||
// Recycle the underlying map.
|
// Recycle the underlying map.
|
||||||
func (m *MSS) Recycle() {
|
func (m *MSS) Recycle() {
|
||||||
if m != nil && *m != nil {
|
if m != nil && *m != nil {
|
||||||
mssPool.Put(map[string]string(*m))
|
mssPool.Put(*m)
|
||||||
*m = nil
|
*m = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -279,15 +280,15 @@ func (b *Bytes) Recycle() {
|
|||||||
// URLValues can be used for url.Values.
|
// URLValues can be used for url.Values.
|
||||||
type URLValues map[string][]string
|
type URLValues map[string][]string
|
||||||
|
|
||||||
var urlValuesPool = sync.Pool{
|
var urlValuesPool = bpool.Pool[map[string][]string]{
|
||||||
New: func() interface{} {
|
New: func() map[string][]string {
|
||||||
return make(map[string][]string, 10)
|
return make(map[string][]string, 10)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewURLValues returns a new URLValues.
|
// NewURLValues returns a new URLValues.
|
||||||
func NewURLValues() *URLValues {
|
func NewURLValues() *URLValues {
|
||||||
u := URLValues(urlValuesPool.Get().(map[string][]string))
|
u := URLValues(urlValuesPool.Get())
|
||||||
return &u
|
return &u
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -342,7 +343,7 @@ func (u *URLValues) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if *u == nil {
|
if *u == nil {
|
||||||
*u = urlValuesPool.Get().(map[string][]string)
|
*u = urlValuesPool.Get()
|
||||||
}
|
}
|
||||||
if len(*u) > 0 {
|
if len(*u) > 0 {
|
||||||
for key := range *u {
|
for key := range *u {
|
||||||
@ -424,9 +425,11 @@ func NewJSONPool[T any]() *JSONPool[T] {
|
|||||||
|
|
||||||
func (p *JSONPool[T]) new() *T {
|
func (p *JSONPool[T]) new() *T {
|
||||||
var zero T
|
var zero T
|
||||||
t := p.pool.Get().(*T)
|
if t, ok := p.pool.Get().(*T); ok {
|
||||||
*t = zero
|
*t = zero
|
||||||
return t
|
return t
|
||||||
|
}
|
||||||
|
return &zero
|
||||||
}
|
}
|
||||||
|
|
||||||
// JSON is a wrapper around a T object that can be serialized.
|
// JSON is a wrapper around a T object that can be serialized.
|
||||||
@ -557,15 +560,15 @@ func (NoPayload) Recycle() {}
|
|||||||
|
|
||||||
// ArrayOf wraps an array of Messagepack compatible objects.
|
// ArrayOf wraps an array of Messagepack compatible objects.
|
||||||
type ArrayOf[T RoundTripper] struct {
|
type ArrayOf[T RoundTripper] struct {
|
||||||
aPool sync.Pool // Arrays
|
aPool sync.Pool // Arrays
|
||||||
ePool sync.Pool // Elements
|
ePool bpool.Pool[T] // Elements
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArrayOf returns a new ArrayOf.
|
// NewArrayOf returns a new ArrayOf.
|
||||||
// You must provide a function that returns a new instance of T.
|
// You must provide a function that returns a new instance of T.
|
||||||
func NewArrayOf[T RoundTripper](newFn func() T) *ArrayOf[T] {
|
func NewArrayOf[T RoundTripper](newFn func() T) *ArrayOf[T] {
|
||||||
return &ArrayOf[T]{
|
return &ArrayOf[T]{
|
||||||
ePool: sync.Pool{New: func() any {
|
ePool: bpool.Pool[T]{New: func() T {
|
||||||
return newFn()
|
return newFn()
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
@ -609,7 +612,7 @@ func (p *ArrayOf[T]) putA(v []T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *ArrayOf[T]) newE() T {
|
func (p *ArrayOf[T]) newE() T {
|
||||||
return p.ePool.Get().(T)
|
return p.ePool.Get()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Array provides a wrapper for an underlying array of serializable objects.
|
// Array provides a wrapper for an underlying array of serializable objects.
|
||||||
|
@ -24,8 +24,9 @@ import (
|
|||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
const defaultFlushInterval = time.Duration(100) * time.Millisecond
|
const defaultFlushInterval = time.Duration(100) * time.Millisecond
|
||||||
@ -53,7 +54,7 @@ func NewForwarder(f *Forwarder) *Forwarder {
|
|||||||
|
|
||||||
type bufPool struct {
|
type bufPool struct {
|
||||||
sz int
|
sz int
|
||||||
pool sync.Pool
|
pool bpool.Pool[*[]byte]
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *bufPool) Put(buf []byte) {
|
func (b *bufPool) Put(buf []byte) {
|
||||||
@ -66,13 +67,16 @@ func (b *bufPool) Put(buf []byte) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (b *bufPool) Get() []byte {
|
func (b *bufPool) Get() []byte {
|
||||||
bufp := b.pool.Get().(*[]byte)
|
bufp := b.pool.Get()
|
||||||
|
if bufp == nil || cap(*bufp) < b.sz {
|
||||||
|
return make([]byte, 0, b.sz)
|
||||||
|
}
|
||||||
return (*bufp)[:b.sz]
|
return (*bufp)[:b.sz]
|
||||||
}
|
}
|
||||||
|
|
||||||
func newBufPool(sz int) httputil.BufferPool {
|
func newBufPool(sz int) httputil.BufferPool {
|
||||||
return &bufPool{sz: sz, pool: sync.Pool{
|
return &bufPool{sz: sz, pool: bpool.Pool[*[]byte]{
|
||||||
New: func() interface{} {
|
New: func() *[]byte {
|
||||||
buf := make([]byte, sz)
|
buf := make([]byte, sz)
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
|
27
internal/http/flush.go
Normal file
27
internal/http/flush.go
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package http
|
||||||
|
|
||||||
|
import "net/http"
|
||||||
|
|
||||||
|
// Flush the ResponseWriter.
|
||||||
|
func Flush(w http.ResponseWriter) {
|
||||||
|
if f, ok := w.(http.Flusher); ok {
|
||||||
|
f.Flush()
|
||||||
|
}
|
||||||
|
}
|
@ -100,13 +100,15 @@ func (listener *httpListener) Addr() (addr net.Addr) {
|
|||||||
return addr
|
return addr
|
||||||
}
|
}
|
||||||
|
|
||||||
tcpAddr := addr.(*net.TCPAddr)
|
if tcpAddr, ok := addr.(*net.TCPAddr); ok {
|
||||||
if ip := net.ParseIP("0.0.0.0"); ip != nil {
|
if ip := net.ParseIP("0.0.0.0"); ip != nil {
|
||||||
tcpAddr.IP = ip
|
tcpAddr.IP = ip
|
||||||
}
|
}
|
||||||
|
|
||||||
addr = tcpAddr
|
addr = tcpAddr
|
||||||
return addr
|
return addr
|
||||||
|
}
|
||||||
|
panic("unknown address type on listener")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Addrs - returns all address information of TCP listeners.
|
// Addrs - returns all address information of TCP listeners.
|
||||||
|
@ -25,10 +25,10 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/disk"
|
"github.com/minio/minio/internal/disk"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -39,28 +39,39 @@ const (
|
|||||||
LargeBlock = 1 * humanize.MiByte // Default r/w block size for normal objects.
|
LargeBlock = 1 * humanize.MiByte // Default r/w block size for normal objects.
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// AlignedBytePool is a pool of fixed size aligned blocks
|
||||||
|
type AlignedBytePool struct {
|
||||||
|
size int
|
||||||
|
p bpool.Pool[*[]byte]
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAlignedBytePool creates a new pool with the specified size.
|
||||||
|
func NewAlignedBytePool(sz int) *AlignedBytePool {
|
||||||
|
return &AlignedBytePool{size: sz, p: bpool.Pool[*[]byte]{New: func() *[]byte {
|
||||||
|
b := disk.AlignedBlock(sz)
|
||||||
|
return &b
|
||||||
|
}}}
|
||||||
|
}
|
||||||
|
|
||||||
// aligned sync.Pool's
|
// aligned sync.Pool's
|
||||||
var (
|
var (
|
||||||
ODirectPoolLarge = sync.Pool{
|
ODirectPoolLarge = NewAlignedBytePool(LargeBlock)
|
||||||
New: func() interface{} {
|
ODirectPoolMedium = NewAlignedBytePool(MediumBlock)
|
||||||
b := disk.AlignedBlock(LargeBlock)
|
ODirectPoolSmall = NewAlignedBytePool(SmallBlock)
|
||||||
return &b
|
|
||||||
},
|
|
||||||
}
|
|
||||||
ODirectPoolMedium = sync.Pool{
|
|
||||||
New: func() interface{} {
|
|
||||||
b := disk.AlignedBlock(MediumBlock)
|
|
||||||
return &b
|
|
||||||
},
|
|
||||||
}
|
|
||||||
ODirectPoolSmall = sync.Pool{
|
|
||||||
New: func() interface{} {
|
|
||||||
b := disk.AlignedBlock(SmallBlock)
|
|
||||||
return &b
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Get a block.
|
||||||
|
func (p *AlignedBytePool) Get() *[]byte {
|
||||||
|
return p.p.Get()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put a block.
|
||||||
|
func (p *AlignedBytePool) Put(pb *[]byte) {
|
||||||
|
if pb != nil && len(*pb) == p.size {
|
||||||
|
p.p.Put(pb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WriteOnCloser implements io.WriteCloser and always
|
// WriteOnCloser implements io.WriteCloser and always
|
||||||
// executes at least one write operation if it is closed.
|
// executes at least one write operation if it is closed.
|
||||||
//
|
//
|
||||||
@ -250,15 +261,6 @@ func NopCloser(w io.Writer) io.WriteCloser {
|
|||||||
return nopCloser{w}
|
return nopCloser{w}
|
||||||
}
|
}
|
||||||
|
|
||||||
const copyBufferSize = 32 * 1024
|
|
||||||
|
|
||||||
var copyBufPool = sync.Pool{
|
|
||||||
New: func() interface{} {
|
|
||||||
b := make([]byte, copyBufferSize)
|
|
||||||
return &b
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// SkipReader skips a given number of bytes and then returns all
|
// SkipReader skips a given number of bytes and then returns all
|
||||||
// remaining data.
|
// remaining data.
|
||||||
type SkipReader struct {
|
type SkipReader struct {
|
||||||
@ -274,12 +276,11 @@ func (s *SkipReader) Read(p []byte) (int, error) {
|
|||||||
}
|
}
|
||||||
if s.skipCount > 0 {
|
if s.skipCount > 0 {
|
||||||
tmp := p
|
tmp := p
|
||||||
if s.skipCount > l && l < copyBufferSize {
|
if s.skipCount > l && l < SmallBlock {
|
||||||
// We may get a very small buffer, so we grab a temporary buffer.
|
// We may get a very small buffer, so we grab a temporary buffer.
|
||||||
bufp := copyBufPool.Get().(*[]byte)
|
bufp := ODirectPoolSmall.Get()
|
||||||
buf := *bufp
|
tmp = *bufp
|
||||||
tmp = buf[:copyBufferSize]
|
defer ODirectPoolSmall.Put(bufp)
|
||||||
defer copyBufPool.Put(bufp)
|
|
||||||
l = int64(len(tmp))
|
l = int64(len(tmp))
|
||||||
}
|
}
|
||||||
for s.skipCount > 0 {
|
for s.skipCount > 0 {
|
||||||
@ -309,7 +310,7 @@ type writerOnly struct {
|
|||||||
|
|
||||||
// Copy is exactly like io.Copy but with reusable buffers.
|
// Copy is exactly like io.Copy but with reusable buffers.
|
||||||
func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
|
func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
|
||||||
bufp := ODirectPoolMedium.Get().(*[]byte)
|
bufp := ODirectPoolMedium.Get()
|
||||||
defer ODirectPoolMedium.Put(bufp)
|
defer ODirectPoolMedium.Put(bufp)
|
||||||
buf := *bufp
|
buf := *bufp
|
||||||
|
|
||||||
|
@ -102,7 +102,7 @@ func TestCloseOnWriter(t *testing.T) {
|
|||||||
|
|
||||||
// Test for AppendFile.
|
// Test for AppendFile.
|
||||||
func TestAppendFile(t *testing.T) {
|
func TestAppendFile(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "")
|
f, err := os.CreateTemp(t.TempDir(), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -111,7 +111,7 @@ func TestAppendFile(t *testing.T) {
|
|||||||
f.WriteString("aaaaaaaaaa")
|
f.WriteString("aaaaaaaaaa")
|
||||||
f.Close()
|
f.Close()
|
||||||
|
|
||||||
f, err = os.CreateTemp("", "")
|
f, err = os.CreateTemp(t.TempDir(), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -162,7 +162,7 @@ func TestSkipReader(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSameFile(t *testing.T) {
|
func TestSameFile(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "")
|
f, err := os.CreateTemp(t.TempDir(), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Error creating tmp file: %v", err)
|
t.Errorf("Error creating tmp file: %v", err)
|
||||||
}
|
}
|
||||||
@ -193,7 +193,7 @@ func TestSameFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyAligned(t *testing.T) {
|
func TestCopyAligned(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "")
|
f, err := os.CreateTemp(t.TempDir(), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Error creating tmp file: %v", err)
|
t.Errorf("Error creating tmp file: %v", err)
|
||||||
}
|
}
|
||||||
@ -202,7 +202,7 @@ func TestCopyAligned(t *testing.T) {
|
|||||||
|
|
||||||
r := strings.NewReader("hello world")
|
r := strings.NewReader("hello world")
|
||||||
|
|
||||||
bufp := ODirectPoolSmall.Get().(*[]byte)
|
bufp := ODirectPoolSmall.Get()
|
||||||
defer ODirectPoolSmall.Put(bufp)
|
defer ODirectPoolSmall.Put(bufp)
|
||||||
|
|
||||||
written, err := CopyAligned(f, io.LimitReader(r, 5), *bufp, r.Size(), f)
|
written, err := CopyAligned(f, io.LimitReader(r, 5), *bufp, r.Size(), f)
|
||||||
|
@ -30,13 +30,13 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash"
|
"hash"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/buger/jsonparser"
|
"github.com/buger/jsonparser"
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
jwtgo "github.com/golang-jwt/jwt/v4"
|
jwtgo "github.com/golang-jwt/jwt/v4"
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SigningMethodHMAC - Implements the HMAC-SHA family of signing methods signing methods
|
// SigningMethodHMAC - Implements the HMAC-SHA family of signing methods signing methods
|
||||||
@ -44,7 +44,7 @@ import (
|
|||||||
type SigningMethodHMAC struct {
|
type SigningMethodHMAC struct {
|
||||||
Name string
|
Name string
|
||||||
Hash crypto.Hash
|
Hash crypto.Hash
|
||||||
HasherPool sync.Pool
|
HasherPool bpool.Pool[hash.Hash]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Specific instances for HS256, HS384, HS512
|
// Specific instances for HS256, HS384, HS512
|
||||||
@ -57,13 +57,13 @@ var (
|
|||||||
const base64BufferSize = 64 * humanize.KiByte
|
const base64BufferSize = 64 * humanize.KiByte
|
||||||
|
|
||||||
var (
|
var (
|
||||||
base64BufPool sync.Pool
|
base64BufPool bpool.Pool[*[]byte]
|
||||||
hmacSigners []*SigningMethodHMAC
|
hmacSigners []*SigningMethodHMAC
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
base64BufPool = sync.Pool{
|
base64BufPool = bpool.Pool[*[]byte]{
|
||||||
New: func() interface{} {
|
New: func() *[]byte {
|
||||||
buf := make([]byte, base64BufferSize)
|
buf := make([]byte, base64BufferSize)
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
@ -76,7 +76,7 @@ func init() {
|
|||||||
}
|
}
|
||||||
for i := range hmacSigners {
|
for i := range hmacSigners {
|
||||||
h := hmacSigners[i].Hash
|
h := hmacSigners[i].Hash
|
||||||
hmacSigners[i].HasherPool.New = func() interface{} {
|
hmacSigners[i].HasherPool.New = func() hash.Hash {
|
||||||
return h.New()
|
return h.New()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -89,13 +89,13 @@ func (s *SigningMethodHMAC) HashBorrower() HashBorrower {
|
|||||||
|
|
||||||
// HashBorrower keeps track of borrowed hashers and allows to return them all.
|
// HashBorrower keeps track of borrowed hashers and allows to return them all.
|
||||||
type HashBorrower struct {
|
type HashBorrower struct {
|
||||||
pool *sync.Pool
|
pool *bpool.Pool[hash.Hash]
|
||||||
borrowed []hash.Hash
|
borrowed []hash.Hash
|
||||||
}
|
}
|
||||||
|
|
||||||
// Borrow a single hasher.
|
// Borrow a single hasher.
|
||||||
func (h *HashBorrower) Borrow() hash.Hash {
|
func (h *HashBorrower) Borrow() hash.Hash {
|
||||||
hasher := h.pool.Get().(hash.Hash)
|
hasher := h.pool.Get()
|
||||||
h.borrowed = append(h.borrowed, hasher)
|
h.borrowed = append(h.borrowed, hasher)
|
||||||
hasher.Reset()
|
hasher.Reset()
|
||||||
return hasher
|
return hasher
|
||||||
@ -323,10 +323,10 @@ func ParseWithStandardClaims(tokenStr string, claims *StandardClaims, key []byte
|
|||||||
return jwtgo.NewValidationError("no key was provided.", jwtgo.ValidationErrorUnverifiable)
|
return jwtgo.NewValidationError("no key was provided.", jwtgo.ValidationErrorUnverifiable)
|
||||||
}
|
}
|
||||||
|
|
||||||
bufp := base64BufPool.Get().(*[]byte)
|
bufp := base64BufPool.Get()
|
||||||
defer base64BufPool.Put(bufp)
|
defer base64BufPool.Put(bufp)
|
||||||
|
|
||||||
tokenBuf := base64BufPool.Get().(*[]byte)
|
tokenBuf := base64BufPool.Get()
|
||||||
defer base64BufPool.Put(tokenBuf)
|
defer base64BufPool.Put(tokenBuf)
|
||||||
|
|
||||||
token := *tokenBuf
|
token := *tokenBuf
|
||||||
@ -419,10 +419,10 @@ func ParseWithClaims(tokenStr string, claims *MapClaims, fn func(*MapClaims) ([]
|
|||||||
return jwtgo.NewValidationError("no Keyfunc was provided.", jwtgo.ValidationErrorUnverifiable)
|
return jwtgo.NewValidationError("no Keyfunc was provided.", jwtgo.ValidationErrorUnverifiable)
|
||||||
}
|
}
|
||||||
|
|
||||||
bufp := base64BufPool.Get().(*[]byte)
|
bufp := base64BufPool.Get()
|
||||||
defer base64BufPool.Put(bufp)
|
defer base64BufPool.Put(bufp)
|
||||||
|
|
||||||
tokenBuf := base64BufPool.Get().(*[]byte)
|
tokenBuf := base64BufPool.Get()
|
||||||
defer base64BufPool.Put(tokenBuf)
|
defer base64BufPool.Put(tokenBuf)
|
||||||
|
|
||||||
token := *tokenBuf
|
token := *tokenBuf
|
||||||
|
@ -26,7 +26,7 @@ func TestIsPresent(t *testing.T) {
|
|||||||
for i, test := range isPresentTests {
|
for i, test := range isPresentTests {
|
||||||
os.Clearenv()
|
os.Clearenv()
|
||||||
for k, v := range test.Env {
|
for k, v := range test.Env {
|
||||||
os.Setenv(k, v)
|
t.Setenv(k, v)
|
||||||
}
|
}
|
||||||
|
|
||||||
ok, err := IsPresent()
|
ok, err := IsPresent()
|
||||||
|
@ -25,7 +25,7 @@ import (
|
|||||||
|
|
||||||
// Test lock fails.
|
// Test lock fails.
|
||||||
func TestLockFail(t *testing.T) {
|
func TestLockFail(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "lock")
|
f, err := os.CreateTemp(t.TempDir(), "lock")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -55,7 +55,7 @@ func TestLockDirFail(t *testing.T) {
|
|||||||
|
|
||||||
// Tests rwlock methods.
|
// Tests rwlock methods.
|
||||||
func TestRWLockedFile(t *testing.T) {
|
func TestRWLockedFile(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "lock")
|
f, err := os.CreateTemp(t.TempDir(), "lock")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@ -118,7 +118,7 @@ func TestRWLockedFile(t *testing.T) {
|
|||||||
|
|
||||||
// Tests lock and unlock semantics.
|
// Tests lock and unlock semantics.
|
||||||
func TestLockAndUnlock(t *testing.T) {
|
func TestLockAndUnlock(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "lock")
|
f, err := os.CreateTemp(t.TempDir(), "lock")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -370,7 +370,6 @@ func lookupLegacyConfigForSubSys(ctx context.Context, subSys string) Config {
|
|||||||
Endpoint: url,
|
Endpoint: url,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
@ -466,7 +466,6 @@ func (h *Target) startQueueProcessor(ctx context.Context, mainWorker bool) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -538,9 +537,11 @@ func New(config Config) (*Target, error) {
|
|||||||
if h.config.Proxy != "" {
|
if h.config.Proxy != "" {
|
||||||
proxyURL, _ := url.Parse(h.config.Proxy)
|
proxyURL, _ := url.Parse(h.config.Proxy)
|
||||||
transport := h.config.Transport
|
transport := h.config.Transport
|
||||||
ctransport := transport.(*http.Transport).Clone()
|
if tr, ok := transport.(*http.Transport); ok {
|
||||||
ctransport.Proxy = http.ProxyURL(proxyURL)
|
ctransport := tr.Clone()
|
||||||
h.config.Transport = ctransport
|
ctransport.Proxy = http.ProxyURL(proxyURL)
|
||||||
|
h.config.Transport = ctransport
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.client = &http.Client{Transport: h.config.Transport}
|
h.client = &http.Client{Transport: h.config.Transport}
|
||||||
|
@ -188,7 +188,6 @@ func (h *Target) startKafkaLogger() {
|
|||||||
// We are not allowed to add when logCh is nil
|
// We are not allowed to add when logCh is nil
|
||||||
h.wg.Add(1)
|
h.wg.Add(1)
|
||||||
defer h.wg.Done()
|
defer h.wg.Done()
|
||||||
|
|
||||||
}
|
}
|
||||||
h.logChMu.RUnlock()
|
h.logChMu.RUnlock()
|
||||||
|
|
||||||
|
@ -56,13 +56,13 @@ func IsLikelyMountPoint(path string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If the directory has a different device as parent, then it is a mountpoint.
|
// If the directory has a different device as parent, then it is a mountpoint.
|
||||||
if s1.Sys().(*syscall.Stat_t).Dev != s2.Sys().(*syscall.Stat_t).Dev {
|
ss1, ok1 := s1.Sys().(*syscall.Stat_t)
|
||||||
// path/.. on a different device as path
|
ss2, ok2 := s2.Sys().(*syscall.Stat_t)
|
||||||
return true
|
return ok1 && ok2 &&
|
||||||
}
|
// path/.. on a different device as path
|
||||||
|
(ss1.Dev != ss2.Dev ||
|
||||||
// path/.. is the same i-node as path - this check is for bind mounts.
|
// path/.. is the same i-node as path - this check is for bind mounts.
|
||||||
return s1.Sys().(*syscall.Stat_t).Ino == s2.Sys().(*syscall.Stat_t).Ino
|
ss1.Ino == ss2.Ino)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CheckCrossDevice - check if any list of paths has any sub-mounts at /proc/mounts.
|
// CheckCrossDevice - check if any list of paths has any sub-mounts at /proc/mounts.
|
||||||
|
@ -40,7 +40,9 @@ var mountPointCache sync.Map
|
|||||||
func IsLikelyMountPoint(path string) bool {
|
func IsLikelyMountPoint(path string) bool {
|
||||||
path = filepath.Dir(path)
|
path = filepath.Dir(path)
|
||||||
if v, ok := mountPointCache.Load(path); ok {
|
if v, ok := mountPointCache.Load(path); ok {
|
||||||
return v.(bool)
|
if b, ok := v.(bool); ok {
|
||||||
|
return b
|
||||||
|
}
|
||||||
}
|
}
|
||||||
wpath, _ := windows.UTF16PtrFromString(path)
|
wpath, _ := windows.UTF16PtrFromString(path)
|
||||||
wvolume := make([]uint16, len(path)+1)
|
wvolume := make([]uint16, len(path)+1)
|
||||||
|
@ -27,25 +27,26 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
csv "github.com/minio/csvparser"
|
csv "github.com/minio/csvparser"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/s3select/sql"
|
"github.com/minio/minio/internal/s3select/sql"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Reader - CSV record reader for S3Select.
|
// Reader - CSV record reader for S3Select.
|
||||||
type Reader struct {
|
type Reader struct {
|
||||||
args *ReaderArgs
|
args *ReaderArgs
|
||||||
readCloser io.ReadCloser // raw input
|
readCloser io.ReadCloser // raw input
|
||||||
buf *bufio.Reader // input to the splitter
|
buf *bufio.Reader // input to the splitter
|
||||||
columnNames []string // names of columns
|
columnNames []string // names of columns
|
||||||
nameIndexMap map[string]int64 // name to column index
|
nameIndexMap map[string]int64 // name to column index
|
||||||
current [][]string // current block of results to be returned
|
current [][]string // current block of results to be returned
|
||||||
recordsRead int // number of records read in current slice
|
recordsRead int // number of records read in current slice
|
||||||
input chan *queueItem // input for workers
|
input chan *queueItem // input for workers
|
||||||
queue chan *queueItem // output from workers in order
|
queue chan *queueItem // output from workers in order
|
||||||
err error // global error state, only touched by Reader.Read
|
err error // global error state, only touched by Reader.Read
|
||||||
bufferPool sync.Pool // pool of []byte objects for input
|
bufferPool bpool.Pool[[]byte] // pool of []byte objects for input
|
||||||
csvDstPool sync.Pool // pool of [][]string used for output
|
csvDstPool bpool.Pool[[][]string] // pool of [][]string used for output
|
||||||
close chan struct{} // used for shutting down the splitter before end of stream
|
close chan struct{} // used for shutting down the splitter before end of stream
|
||||||
readerWg sync.WaitGroup // used to keep track of async reader.
|
readerWg sync.WaitGroup // used to keep track of async reader.
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueItem is an item in the queue.
|
// queueItem is an item in the queue.
|
||||||
@ -69,7 +70,7 @@ func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
|
|||||||
r.err = io.EOF
|
r.err = io.EOF
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
//nolint:staticcheck // SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
|
|
||||||
r.csvDstPool.Put(r.current)
|
r.csvDstPool.Put(r.current)
|
||||||
r.current = <-item.dst
|
r.current = <-item.dst
|
||||||
r.err = item.err
|
r.err = item.err
|
||||||
@ -182,12 +183,12 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
r.bufferPool.New = func() interface{} {
|
r.bufferPool.New = func() []byte {
|
||||||
return make([]byte, csvSplitSize+1024)
|
return make([]byte, csvSplitSize+1024)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return first block
|
// Return first block
|
||||||
next, nextErr := r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
|
next, nextErr := r.nextSplit(csvSplitSize, r.bufferPool.Get())
|
||||||
// Check if first block is valid.
|
// Check if first block is valid.
|
||||||
if !utf8.Valid(next) {
|
if !utf8.Valid(next) {
|
||||||
return errInvalidTextEncodingError()
|
return errInvalidTextEncodingError()
|
||||||
@ -224,7 +225,7 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
|
|||||||
// Exit on any error.
|
// Exit on any error.
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
next, nextErr = r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
|
next, nextErr = r.nextSplit(csvSplitSize, r.bufferPool.Get())
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@ -236,8 +237,8 @@ func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
|
|||||||
in.dst <- nil
|
in.dst <- nil
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
dst, ok := r.csvDstPool.Get().([][]string)
|
dst := r.csvDstPool.Get()
|
||||||
if !ok {
|
if len(dst) < 1000 {
|
||||||
dst = make([][]string, 0, 1000)
|
dst = make([][]string, 0, 1000)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -24,6 +24,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/s3select/jstream"
|
"github.com/minio/minio/internal/s3select/jstream"
|
||||||
"github.com/minio/minio/internal/s3select/sql"
|
"github.com/minio/minio/internal/s3select/sql"
|
||||||
)
|
)
|
||||||
@ -32,17 +33,17 @@ import (
|
|||||||
// Operates concurrently on line-delimited JSON.
|
// Operates concurrently on line-delimited JSON.
|
||||||
type PReader struct {
|
type PReader struct {
|
||||||
args *ReaderArgs
|
args *ReaderArgs
|
||||||
readCloser io.ReadCloser // raw input
|
readCloser io.ReadCloser // raw input
|
||||||
buf *bufio.Reader // input to the splitter
|
buf *bufio.Reader // input to the splitter
|
||||||
current []jstream.KVS // current block of results to be returned
|
current []jstream.KVS // current block of results to be returned
|
||||||
recordsRead int // number of records read in current slice
|
recordsRead int // number of records read in current slice
|
||||||
input chan *queueItem // input for workers
|
input chan *queueItem // input for workers
|
||||||
queue chan *queueItem // output from workers in order
|
queue chan *queueItem // output from workers in order
|
||||||
err error // global error state, only touched by Reader.Read
|
err error // global error state, only touched by Reader.Read
|
||||||
bufferPool sync.Pool // pool of []byte objects for input
|
bufferPool bpool.Pool[[]byte] // pool of []byte objects for input
|
||||||
kvDstPool sync.Pool // pool of []jstream.KV used for output
|
kvDstPool bpool.Pool[[]jstream.KVS] // pool of []jstream.KVS used for output
|
||||||
close chan struct{} // used for shutting down the splitter before end of stream
|
close chan struct{} // used for shutting down the splitter before end of stream
|
||||||
readerWg sync.WaitGroup // used to keep track of async reader.
|
readerWg sync.WaitGroup // used to keep track of async reader.
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueItem is an item in the queue.
|
// queueItem is an item in the queue.
|
||||||
@ -66,7 +67,6 @@ func (r *PReader) Read(dst sql.Record) (sql.Record, error) {
|
|||||||
r.err = io.EOF
|
r.err = io.EOF
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
//nolint:staticcheck // SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
|
|
||||||
r.kvDstPool.Put(r.current)
|
r.kvDstPool.Put(r.current)
|
||||||
r.current = <-item.dst
|
r.current = <-item.dst
|
||||||
r.err = item.err
|
r.err = item.err
|
||||||
@ -133,7 +133,7 @@ const jsonSplitSize = 128 << 10
|
|||||||
// and a number of workers based on GOMAXPROCS.
|
// and a number of workers based on GOMAXPROCS.
|
||||||
// If an error is returned no goroutines have been started and r.err will have been set.
|
// If an error is returned no goroutines have been started and r.err will have been set.
|
||||||
func (r *PReader) startReaders() {
|
func (r *PReader) startReaders() {
|
||||||
r.bufferPool.New = func() interface{} {
|
r.bufferPool.New = func() []byte {
|
||||||
return make([]byte, jsonSplitSize+1024)
|
return make([]byte, jsonSplitSize+1024)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -148,7 +148,7 @@ func (r *PReader) startReaders() {
|
|||||||
defer close(r.queue)
|
defer close(r.queue)
|
||||||
defer r.readerWg.Done()
|
defer r.readerWg.Done()
|
||||||
for {
|
for {
|
||||||
next, err := r.nextSplit(jsonSplitSize, r.bufferPool.Get().([]byte))
|
next, err := r.nextSplit(jsonSplitSize, r.bufferPool.Get())
|
||||||
q := queueItem{
|
q := queueItem{
|
||||||
input: next,
|
input: next,
|
||||||
dst: make(chan []jstream.KVS, 1),
|
dst: make(chan []jstream.KVS, 1),
|
||||||
@ -180,8 +180,8 @@ func (r *PReader) startReaders() {
|
|||||||
in.dst <- nil
|
in.dst <- nil
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
dst, ok := r.kvDstPool.Get().([]jstream.KVS)
|
dst := r.kvDstPool.Get()
|
||||||
if !ok {
|
if len(dst) < 1000 {
|
||||||
dst = make([]jstream.KVS, 0, 1000)
|
dst = make([]jstream.KVS, 0, 1000)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,7 +193,7 @@ func (r *PReader) startReaders() {
|
|||||||
if mv.ValueType == jstream.Object {
|
if mv.ValueType == jstream.Object {
|
||||||
// This is a JSON object type (that preserves key
|
// This is a JSON object type (that preserves key
|
||||||
// order)
|
// order)
|
||||||
kvs = mv.Value.(jstream.KVS)
|
kvs, _ = mv.Value.(jstream.KVS)
|
||||||
} else {
|
} else {
|
||||||
// To be AWS S3 compatible Select for JSON needs to
|
// To be AWS S3 compatible Select for JSON needs to
|
||||||
// output non-object JSON as single column value
|
// output non-object JSON as single column value
|
||||||
|
@ -51,7 +51,7 @@ func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
|
|||||||
if v.ValueType == jstream.Object {
|
if v.ValueType == jstream.Object {
|
||||||
// This is a JSON object type (that preserves key
|
// This is a JSON object type (that preserves key
|
||||||
// order)
|
// order)
|
||||||
kvs = v.Value.(jstream.KVS)
|
kvs, _ = v.Value.(jstream.KVS)
|
||||||
} else {
|
} else {
|
||||||
// To be AWS S3 compatible Select for JSON needs to
|
// To be AWS S3 compatible Select for JSON needs to
|
||||||
// output non-object JSON as single column value
|
// output non-object JSON as single column value
|
||||||
|
@ -26,6 +26,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A message is in the format specified in
|
// A message is in the format specified in
|
||||||
@ -262,7 +264,7 @@ func (writer *messageWriter) write(data []byte) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
writer.writer.(http.Flusher).Flush()
|
xhttp.Flush(writer.writer)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -56,7 +56,6 @@ func (pr *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
|
|||||||
|
|
||||||
kvs := jstream.KVS{}
|
kvs := jstream.KVS{}
|
||||||
for _, col := range pr.r.Columns() {
|
for _, col := range pr.r.Columns() {
|
||||||
|
|
||||||
var value interface{}
|
var value interface{}
|
||||||
if v, ok := nextRow[col.FlatName()]; ok {
|
if v, ok := nextRow[col.FlatName()]; ok {
|
||||||
value, err = convertFromAnnotation(col.Element(), v)
|
value, err = convertFromAnnotation(col.Element(), v)
|
||||||
|
@ -32,6 +32,7 @@ import (
|
|||||||
"github.com/klauspost/compress/s2"
|
"github.com/klauspost/compress/s2"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
gzip "github.com/klauspost/pgzip"
|
gzip "github.com/klauspost/pgzip"
|
||||||
|
"github.com/minio/minio/internal/bpool"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
xioutil "github.com/minio/minio/internal/ioutil"
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/minio/internal/s3select/csv"
|
"github.com/minio/minio/internal/s3select/csv"
|
||||||
@ -81,15 +82,15 @@ func init() {
|
|||||||
parquetSupport = env.Get("MINIO_API_SELECT_PARQUET", config.EnableOff) == config.EnableOn
|
parquetSupport = env.Get("MINIO_API_SELECT_PARQUET", config.EnableOff) == config.EnableOn
|
||||||
}
|
}
|
||||||
|
|
||||||
var bufPool = sync.Pool{
|
var bufPool = bpool.Pool[*bytes.Buffer]{
|
||||||
New: func() interface{} {
|
New: func() *bytes.Buffer {
|
||||||
// make a buffer with a reasonable capacity.
|
// make a buffer with a reasonable capacity.
|
||||||
return bytes.NewBuffer(make([]byte, 0, maxRecordSize))
|
return bytes.NewBuffer(make([]byte, 0, maxRecordSize))
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var bufioWriterPool = sync.Pool{
|
var bufioWriterPool = bpool.Pool[*bufio.Writer]{
|
||||||
New: func() interface{} {
|
New: func() *bufio.Writer {
|
||||||
// io.Discard is just used to create the writer. Actual destination
|
// io.Discard is just used to create the writer. Actual destination
|
||||||
// writer is set later by Reset() before using it.
|
// writer is set later by Reset() before using it.
|
||||||
return bufio.NewWriter(xioutil.Discard)
|
return bufio.NewWriter(xioutil.Discard)
|
||||||
@ -468,7 +469,7 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
|
|||||||
switch s3Select.Output.format {
|
switch s3Select.Output.format {
|
||||||
case csvFormat:
|
case csvFormat:
|
||||||
// Use bufio Writer to prevent csv.Writer from allocating a new buffer.
|
// Use bufio Writer to prevent csv.Writer from allocating a new buffer.
|
||||||
bufioWriter := bufioWriterPool.Get().(*bufio.Writer)
|
bufioWriter := bufioWriterPool.Get()
|
||||||
defer func() {
|
defer func() {
|
||||||
bufioWriter.Reset(xioutil.Discard)
|
bufioWriter.Reset(xioutil.Discard)
|
||||||
bufioWriterPool.Put(bufioWriter)
|
bufioWriterPool.Put(bufioWriter)
|
||||||
@ -530,7 +531,7 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
|
|||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
sendRecord := func() bool {
|
sendRecord := func() bool {
|
||||||
buf := bufPool.Get().(*bytes.Buffer)
|
buf := bufPool.Get()
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
|
|
||||||
for _, outputRecord := range outputQueue {
|
for _, outputRecord := range outputQueue {
|
||||||
|
@ -107,7 +107,6 @@ func (e *FuncExpr) evalSQLFnNode(r Record, tableAlias string) (res *Value, err e
|
|||||||
|
|
||||||
case sqlFnDateDiff:
|
case sqlFnDateDiff:
|
||||||
return handleDateDiff(r, e.DateDiff, tableAlias)
|
return handleDateDiff(r, e.DateDiff, tableAlias)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// For all simple argument functions, we evaluate the arguments here
|
// For all simple argument functions, we evaluate the arguments here
|
||||||
|
@ -107,7 +107,6 @@ func evalSQLLike(text, pattern string, escape rune) (match bool, err error) {
|
|||||||
default:
|
default:
|
||||||
s = append(s, r)
|
s = append(s, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
if hasLeadingPercent {
|
if hasLeadingPercent {
|
||||||
return strings.HasSuffix(text, string(s)), nil
|
return strings.HasSuffix(text, string(s)), nil
|
||||||
|
@ -175,7 +175,6 @@ func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) {
|
|||||||
seconds := duration / time.Second
|
seconds := duration / time.Second
|
||||||
return FromInt(int64(seconds)), nil
|
return FromInt(int64(seconds)), nil
|
||||||
default:
|
default:
|
||||||
|
|
||||||
}
|
}
|
||||||
return nil, errNotImplemented
|
return nil, errNotImplemented
|
||||||
}
|
}
|
||||||
|
@ -663,8 +663,13 @@ func inferTypeForArithOp(a *Value) error {
|
|||||||
a.setFloat(f)
|
a.setFloat(f)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var s string
|
||||||
err := fmt.Errorf("Could not convert %q to a number", string(a.value.([]byte)))
|
if v, ok := a.value.([]byte); ok {
|
||||||
|
s = string(v)
|
||||||
|
} else {
|
||||||
|
s = fmt.Sprint(a.value)
|
||||||
|
}
|
||||||
|
err := fmt.Errorf("Could not convert %q to a number", s)
|
||||||
return errInvalidDataType(err)
|
return errInvalidDataType(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user