mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
Write xml.Header first instead of spaces to handle XML parsers (#7253)
Clients like AWS SDK Java and AWS cli XML parsers are unable to handle on `\r\n` characters to avoid these errors send XML header first and write white space characters instead. Also handle cases to avoid double WriteHeader calls
This commit is contained in:
parent
2232b0b55f
commit
bedcb7442a
@ -1,60 +0,0 @@
|
|||||||
/*
|
|
||||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package cmd
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Represents additional fields necessary for ErrPartTooSmall S3 error.
|
|
||||||
type completeMultipartAPIError struct {
|
|
||||||
// Proposed size represents uploaded size of the part.
|
|
||||||
ProposedSize int64
|
|
||||||
// Minimum size allowed epresents the minimum size allowed per
|
|
||||||
// part. Defaults to 5MB.
|
|
||||||
MinSizeAllowed int64
|
|
||||||
// Part number of the part which is incorrect.
|
|
||||||
PartNumber int
|
|
||||||
// ETag of the part which is incorrect.
|
|
||||||
PartETag string
|
|
||||||
// Other default XML error responses.
|
|
||||||
APIErrorResponse
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeErrorResponsePartTooSmall - function is used specifically to
|
|
||||||
// construct a proper error response during CompleteMultipartUpload
|
|
||||||
// when one of the parts is < 5MB.
|
|
||||||
// The requirement comes due to the fact that generic ErrorResponse
|
|
||||||
// XML doesn't carry the additional fields required to send this
|
|
||||||
// error. So we construct a new type which lies well within the scope
|
|
||||||
// of this function.
|
|
||||||
func writePartSmallErrorResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, err PartTooSmall) {
|
|
||||||
apiError := toAPIError(ctx, err)
|
|
||||||
// Generate complete multipart error response.
|
|
||||||
errorResponse := getAPIErrorResponse(ctx, apiError, r.URL.Path,
|
|
||||||
w.Header().Get(responseRequestIDKey),
|
|
||||||
w.Header().Get(responseDeploymentIDKey))
|
|
||||||
cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
|
|
||||||
encodedErrorResponse := encodeResponse(cmpErrResp)
|
|
||||||
|
|
||||||
// respond with 400 bad request.
|
|
||||||
w.WriteHeader(apiError.HTTPStatusCode)
|
|
||||||
// Write error body.
|
|
||||||
w.Write(encodedErrorResponse)
|
|
||||||
w.(http.Flusher).Flush()
|
|
||||||
}
|
|
@ -587,7 +587,8 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Generate error response.
|
// Generate error response.
|
||||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
||||||
|
w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
||||||
encodedErrorResponse := encodeResponse(errorResponse)
|
encodedErrorResponse := encodeResponse(errorResponse)
|
||||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
||||||
}
|
}
|
||||||
|
@ -2152,27 +2152,52 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
|
|||||||
writeSuccessResponseXML(w, encodedSuccessResponse)
|
writeSuccessResponseXML(w, encodedSuccessResponse)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send white spaces every 10 seconds to the client till completeMultiPartUpload() is done so that the client does not time out.
|
type whiteSpaceWriter struct {
|
||||||
// Downside is we might send 200OK and then send error XML. But accoording to S3 spec
|
http.ResponseWriter
|
||||||
// the client is supposed to check for error XML even if it received 200OK. But for erasure this is not a
|
http.Flusher
|
||||||
// problem as completeMultiPartUpload() is quick. Even For FS, it would not be an issue
|
written bool
|
||||||
// as we do background append as and when the parts arrive and completeMultiPartUpload is quick.
|
}
|
||||||
// Only in a rare case where parts would be out of order will FS' completeMultiPartUpload() take a longer time.
|
|
||||||
func sendWhiteSpace(ctx context.Context, w http.ResponseWriter) <-chan struct{} {
|
func (w *whiteSpaceWriter) Write(b []byte) (n int, err error) {
|
||||||
doneCh := make(chan struct{})
|
n, err = w.ResponseWriter.Write(b)
|
||||||
|
w.written = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *whiteSpaceWriter) WriteHeader(statusCode int) {
|
||||||
|
if !w.written {
|
||||||
|
w.ResponseWriter.WriteHeader(statusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send empty whitespaces every 10 seconds to the client till completeMultiPartUpload() is
|
||||||
|
// done so that the client does not time out. Downside is we might send 200 OK and
|
||||||
|
// then send error XML. But accoording to S3 spec the client is supposed to check
|
||||||
|
// for error XML even if it received 200 OK. But for erasure this is not a problem
|
||||||
|
// as completeMultiPartUpload() is quick. Even For FS, it would not be an issue as
|
||||||
|
// we do background append as and when the parts arrive and completeMultiPartUpload
|
||||||
|
// is quick. Only in a rare case where parts would be out of order will
|
||||||
|
// FS:completeMultiPartUpload() take a longer time.
|
||||||
|
func sendWhiteSpace(ctx context.Context, w http.ResponseWriter) <-chan bool {
|
||||||
|
doneCh := make(chan bool)
|
||||||
go func() {
|
go func() {
|
||||||
ticker := time.NewTicker(time.Second * 10)
|
ticker := time.NewTicker(time.Second * 10)
|
||||||
|
headerWritten := false
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
// Write a white space char to the client to prevent client timeouts
|
// Write header if not written yet.
|
||||||
// when the server takes a long time to completeMultiPartUpload()
|
if !headerWritten {
|
||||||
if _, err := w.Write([]byte("\n\r")); err != nil {
|
w.Write([]byte(xml.Header))
|
||||||
logger.LogIf(ctx, err)
|
headerWritten = true
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Once header is written keep writing empty spaces
|
||||||
|
// which are ignored by client SDK XML parsers.
|
||||||
|
// This occurs when server takes long time to completeMultiPartUpload()
|
||||||
|
w.Write([]byte(" "))
|
||||||
w.(http.Flusher).Flush()
|
w.(http.Flusher).Flush()
|
||||||
case doneCh <- struct{}{}:
|
case doneCh <- headerWritten:
|
||||||
ticker.Stop()
|
ticker.Stop()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -2309,18 +2334,36 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
|||||||
if api.CacheAPI() != nil {
|
if api.CacheAPI() != nil {
|
||||||
completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
|
completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This code is specifically to handle the requirements for slow
|
||||||
|
// complete multipart upload operations on FS mode.
|
||||||
|
writeErrorResponseWithoutXMLHeader := func(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
|
||||||
|
switch err.Code {
|
||||||
|
case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
|
||||||
|
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||||
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||||
|
w.Header().Set("Retry-After", "120")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate error response.
|
||||||
|
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
||||||
|
w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
|
||||||
|
encodedErrorResponse, _ := xml.Marshal(errorResponse)
|
||||||
|
setCommonHeaders(w)
|
||||||
|
w.Header().Set("Content-Type", string(mimeXML))
|
||||||
|
w.Write(encodedErrorResponse)
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
}
|
||||||
|
w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
|
||||||
completeDoneCh := sendWhiteSpace(ctx, w)
|
completeDoneCh := sendWhiteSpace(ctx, w)
|
||||||
objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
|
objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
|
||||||
// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
|
// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
|
||||||
// can cause white space to be written after we send XML response in a race condition.
|
// can cause white space to be written after we send XML response in a race condition.
|
||||||
<-completeDoneCh
|
headerWritten := <-completeDoneCh
|
||||||
if err != nil {
|
if err != nil {
|
||||||
switch oErr := err.(type) {
|
if headerWritten {
|
||||||
case PartTooSmall:
|
writeErrorResponseWithoutXMLHeader(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
// Write part too small error.
|
} else {
|
||||||
writePartSmallErrorResponse(ctx, w, r, oErr)
|
|
||||||
default:
|
|
||||||
// Handle all other generic issues.
|
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@ -2330,10 +2373,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
|||||||
location := getObjectLocation(r, globalDomainName, bucket, object)
|
location := getObjectLocation(r, globalDomainName, bucket, object)
|
||||||
// Generate complete multipart response.
|
// Generate complete multipart response.
|
||||||
response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo.ETag)
|
response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo.ETag)
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
var encodedSuccessResponse []byte
|
||||||
if err != nil {
|
if !headerWritten {
|
||||||
writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
|
encodedSuccessResponse = encodeResponse(response)
|
||||||
return
|
} else {
|
||||||
|
encodedSuccessResponse, err = xml.Marshal(response)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseWithoutXMLHeader(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set etag.
|
// Set etag.
|
||||||
|
@ -2740,9 +2740,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
|||||||
accessKey: credentials.AccessKey,
|
accessKey: credentials.AccessKey,
|
||||||
secretKey: credentials.SecretKey,
|
secretKey: credentials.SecretKey,
|
||||||
|
|
||||||
expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
|
expectedContent: encodeResponse(getAPIErrorResponse(ctx,
|
||||||
getAPIErrorResponse(ctx, toAPIError(ctx, PartTooSmall{PartNumber: 1}),
|
toAPIError(ctx, PartTooSmall{PartNumber: 1}),
|
||||||
getGetObjectURL("", bucketName, objectName), "", "")}),
|
getGetObjectURL("", bucketName, objectName), "", "")),
|
||||||
expectedRespStatus: http.StatusBadRequest,
|
expectedRespStatus: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
// Test case - 5.
|
// Test case - 5.
|
||||||
|
Loading…
Reference in New Issue
Block a user