diff --git a/.github/workflows/run-mint.sh b/.github/workflows/run-mint.sh index eafd69d03..6a64465c9 100755 --- a/.github/workflows/run-mint.sh +++ b/.github/workflows/run-mint.sh @@ -7,6 +7,7 @@ export ACCESS_KEY="$2" export SECRET_KEY="$3" export JOB_NAME="$4" export MINT_MODE="full" +export MINT_NO_FULL_OBJECT="true" docker system prune -f || true docker volume prune -f || true @@ -35,6 +36,7 @@ docker run --rm --net=mint_default \ -e ACCESS_KEY="${ACCESS_KEY}" \ -e SECRET_KEY="${SECRET_KEY}" \ -e ENABLE_HTTPS=0 \ + -e MINT_NO_FULL_OBJECT="${MINT_NO_FULL_OBJECT}" \ -e MINT_MODE="${MINT_MODE}" \ docker.io/minio/mint:edge diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 6db63a042..b61c4d900 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -1338,6 +1338,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h return } } + opts.EncryptFn = metadataEncrypter(objectEncryptionKey) pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index 7374f6c65..ff0a88279 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -404,12 +404,7 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin metadata = make(map[string]string) } - wantCRC, err := hash.GetContentChecksum(hdr) - if err != nil { - return opts, fmt.Errorf("invalid/unknown checksum sent: %v", err) - } etag := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceETag)) - if crypto.S3KMS.IsRequested(hdr) { keyID, context, err := crypto.S3KMS.ParseHTTP(hdr) if err != nil { @@ -423,7 +418,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin ServerSideEncryption: sseKms, UserDefined: metadata, MTime: mtime, - WantChecksum: wantCRC, PreserveETag: etag, }, nil } @@ -438,7 +432,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin opts.ReplicationSourceRetentionTimestamp = retaintimestmp opts.ReplicationSourceTaggingTimestamp = taggingtimestmp opts.PreserveETag = etag - opts.WantChecksum = wantCRC return opts, nil } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index b99fb7e00..e017e6777 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1899,6 +1899,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req var reader io.Reader reader = rd + var opts ObjectOptions + opts, err = putOptsFromReq(ctx, r, bucket, object, metadata) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + actualSize := size var idxCb func() []byte if isCompressible(r.Header, object) && size > minCompressibleSize { @@ -1915,6 +1922,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } + opts.WantChecksum = actualReader.Checksum() + // Set compression metrics. var s2c io.ReadCloser wantEncryption := crypto.Requested(r.Header) @@ -1945,22 +1954,17 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } - if err := hashReader.AddChecksum(r, size < 0); err != nil { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) - return + if size >= 0 { + if err := hashReader.AddChecksum(r, false); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + opts.WantChecksum = hashReader.Checksum() } rawReader := hashReader pReader := NewPutObjReader(rawReader) - - var opts ObjectOptions - opts, err = putOptsFromReq(ctx, r, bucket, object, metadata) - if err != nil { - writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) - return - } opts.IndexCB = idxCb - opts.WantChecksum = hashReader.Checksum() if opts.PreserveETag != "" || r.Header.Get(xhttp.IfMatch) != "" || diff --git a/internal/hash/reader.go b/internal/hash/reader.go index 272452f06..caca03186 100644 --- a/internal/hash/reader.go +++ b/internal/hash/reader.go @@ -257,6 +257,26 @@ func (r *Reader) Read(p []byte) (int, error) { r.contentHasher.Write(p[:n]) } + // If we have reached our expected size, + // do one more read to ensure we are at EOF + // and that any trailers have been read. + attempts := 0 + for err == nil && r.size >= 0 && r.bytesRead >= r.size { + attempts++ + if r.bytesRead > r.size { + return 0, SizeTooLarge{Want: r.size, Got: r.bytesRead} + } + var tmp [1]byte + var n2 int + n2, err = r.src.Read(tmp[:]) + if n2 > 0 { + return 0, SizeTooLarge{Want: r.size, Got: r.bytesRead} + } + if attempts == 100 { + return 0, io.ErrNoProgress + } + } + if err == io.EOF { // Verify content SHA256, if set. if r.expectedMin > 0 { if r.bytesRead < r.expectedMin {