From 734e779b198f4a4a6c8564d59de5e5d7a1493673 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 1 Jul 2016 14:33:28 -0700 Subject: [PATCH] XL/erasureCreate: Create a limit reader if size is specified. (#2059) This is needed so that we only write data which was requested for, using a limit reader avoids spurious reads on the incoming client data. Additionally using limit reader provides server safety from rogue clients sending copious amounts of data (for example a denial of service attack). This patch also caters for size == -1 when content encoding from a client is set as chunked, we happily read till io.EOF --- erasure-createfile.go | 27 +++++++++++++++------------ xl-v1-multipart.go | 15 +++++++++++++-- xl-v1-object.go | 15 +++++++++++++-- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/erasure-createfile.go b/erasure-createfile.go index 8ae84e902..ffe965947 100644 --- a/erasure-createfile.go +++ b/erasure-createfile.go @@ -38,39 +38,42 @@ func erasureCreateFile(disks []StorageAPI, volume string, path string, partName // Read until io.EOF, erasure codes data and writes to all disks. for { - var n int var blocks [][]byte - n, err = io.ReadFull(data, buf) - if err == io.EOF { + n, rErr := io.ReadFull(data, buf) + // FIXME: this is a bug in Golang, n == 0 and err == + // io.ErrUnexpectedEOF for io.ReadFull function. + if n == 0 && rErr == io.ErrUnexpectedEOF { + return nil, 0, rErr + } + if rErr == io.EOF { // We have reached EOF on the first byte read, io.Reader // must be 0bytes, we don't need to erasure code // data. Will create a 0byte file instead. if size == 0 { blocks = make([][]byte, len(disks)) - err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum) - if err != nil { - return nil, 0, err + rErr = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum) + if rErr != nil { + return nil, 0, rErr } } // else we have reached EOF after few reads, no need to // add an additional 0bytes at the end. break } - if err != nil && err != io.ErrUnexpectedEOF { - return nil, 0, err + if rErr != nil && rErr != io.ErrUnexpectedEOF { + return nil, 0, rErr } - size += int64(n) // Returns encoded blocks. var enErr error - blocks, enErr = encodeData(buf[:n], eInfo.DataBlocks, eInfo.ParityBlocks) + blocks, enErr = encodeData(buf[0:n], eInfo.DataBlocks, eInfo.ParityBlocks) if enErr != nil { return nil, 0, enErr } // Write to all disks. - err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum) - if err != nil { + if err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum); err != nil { return nil, 0, err } + size += int64(n) } // Save the checksums. diff --git a/xl-v1-multipart.go b/xl-v1-multipart.go index ba462b625..b931aed0d 100644 --- a/xl-v1-multipart.go +++ b/xl-v1-multipart.go @@ -340,6 +340,13 @@ func (xl xlObjects) putObjectPart(bucket string, object string, uploadID string, // Initialize md5 writer. md5Writer := md5.New() + // Limit the reader to its provided size > 0. + if size > 0 { + // This is done so that we can avoid erroneous clients sending + // more data than the set content size. + data = io.LimitReader(data, size+1) + } // else we read till EOF. + // Construct a tee reader for md5sum. teeReader := io.TeeReader(data, md5Writer) @@ -350,13 +357,17 @@ func (xl xlObjects) putObjectPart(bucket string, object string, uploadID string, } // Erasure code data and write across all disks. - newEInfos, n, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, partSuffix, teeReader, eInfos, xl.writeQuorum) + newEInfos, sizeWritten, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, partSuffix, teeReader, eInfos, xl.writeQuorum) if err != nil { return "", toObjectErr(err, minioMetaBucket, tmpPartPath) } + + // For size == -1, perhaps client is sending in chunked encoding + // set the size as size that was actually written. if size == -1 { - size = n + size = sizeWritten } + // Calculate new md5sum. newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) if md5Hex != "" { diff --git a/xl-v1-object.go b/xl-v1-object.go index c2ea20105..dacc60fca 100644 --- a/xl-v1-object.go +++ b/xl-v1-object.go @@ -332,6 +332,13 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // Initialize md5 writer. md5Writer := md5.New() + // Limit the reader to its provided size if specified. + if size > 0 { + // This is done so that we can avoid erroneous clients sending + // more data than the set content size. + data = io.LimitReader(data, size+1) + } // else we read till EOF. + // Tee reader combines incoming data stream and md5, data read // from input stream is written to md5. teeReader := io.TeeReader(data, md5Writer) @@ -343,13 +350,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. } // Erasure code and write across all disks. - newEInfos, n, err := erasureCreateFile(onlineDisks, minioMetaBucket, tempErasureObj, "part.1", teeReader, eInfos, xl.writeQuorum) + newEInfos, sizeWritten, err := erasureCreateFile(onlineDisks, minioMetaBucket, tempErasureObj, "part.1", teeReader, eInfos, xl.writeQuorum) if err != nil { return "", toObjectErr(err, minioMetaBucket, tempErasureObj) } + + // For size == -1, perhaps client is sending in chunked encoding + // set the size as size that was actually written. if size == -1 { - size = n + size = sizeWritten } + // Save additional erasureMetadata. modTime := time.Now().UTC()