XL/erasureCreate: Create a limit reader if size is specified. (#2059)

This is needed so that we only write data which was requested
for, using a limit reader avoids spurious reads on the incoming
client data. Additionally using limit reader provides server
safety from rogue clients sending copious amounts of data (for
example a denial of service attack).

This patch also caters for size == -1 when content encoding from
a client is set as chunked, we happily read till io.EOF

erasure-createfile.go

@@ -38,39 +38,42 @@ func erasureCreateFile(disks []StorageAPI, volume string, path string, partName
 
 	// Read until io.EOF, erasure codes data and writes to all disks.
 	for {
-		var n int
 		var blocks [][]byte
-		n, err = io.ReadFull(data, buf)
-		if err == io.EOF {
+		n, rErr := io.ReadFull(data, buf)
+		// FIXME: this is a bug in Golang, n == 0 and err ==
+		// io.ErrUnexpectedEOF for io.ReadFull function.
+		if n == 0 && rErr == io.ErrUnexpectedEOF {
+			return nil, 0, rErr
+		}
+		if rErr == io.EOF {
 			// We have reached EOF on the first byte read, io.Reader
 			// must be 0bytes, we don't need to erasure code
 			// data. Will create a 0byte file instead.
 			if size == 0 {
 				blocks = make([][]byte, len(disks))
-				err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum)
-				if err != nil {
-					return nil, 0, err
+				rErr = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum)
+				if rErr != nil {
+					return nil, 0, rErr
 				}
 			} // else we have reached EOF after few reads, no need to
 			// add an additional 0bytes at the end.
 			break
 		}
-		if err != nil && err != io.ErrUnexpectedEOF {
-			return nil, 0, err
+		if rErr != nil && rErr != io.ErrUnexpectedEOF {
+			return nil, 0, rErr
 		}
-		size += int64(n)
 		// Returns encoded blocks.
 		var enErr error
-		blocks, enErr = encodeData(buf[:n], eInfo.DataBlocks, eInfo.ParityBlocks)
+		blocks, enErr = encodeData(buf[0:n], eInfo.DataBlocks, eInfo.ParityBlocks)
 		if enErr != nil {
 			return nil, 0, enErr
 		}
 
 		// Write to all disks.
-		err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum)
-		if err != nil {
+		if err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum); err != nil {
 			return nil, 0, err
 		}
+		size += int64(n)
 	}
 
 	// Save the checksums.

xl-v1-multipart.go

@@ -340,6 +340,13 @@ func (xl xlObjects) putObjectPart(bucket string, object string, uploadID string,
 	// Initialize md5 writer.
 	md5Writer := md5.New()
 
+	// Limit the reader to its provided size > 0.
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending
+		// more data than the set content size.
+		data = io.LimitReader(data, size+1)
+	} // else we read till EOF.
+
 	// Construct a tee reader for md5sum.
 	teeReader := io.TeeReader(data, md5Writer)
 
@@ -350,13 +357,17 @@ func (xl xlObjects) putObjectPart(bucket string, object string, uploadID string,
 	}
 
 	// Erasure code data and write across all disks.
-	newEInfos, n, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, partSuffix, teeReader, eInfos, xl.writeQuorum)
+	newEInfos, sizeWritten, err := erasureCreateFile(onlineDisks, minioMetaBucket, tmpPartPath, partSuffix, teeReader, eInfos, xl.writeQuorum)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaBucket, tmpPartPath)
 	}
+
+	// For size == -1, perhaps client is sending in chunked encoding
+	// set the size as size that was actually written.
 	if size == -1 {
-		size = n
+		size = sizeWritten
 	}
+
 	// Calculate new md5sum.
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	if md5Hex != "" {

xl-v1-object.go

@@ -332,6 +332,13 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.
 	// Initialize md5 writer.
 	md5Writer := md5.New()
 
+	// Limit the reader to its provided size if specified.
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending
+		// more data than the set content size.
+		data = io.LimitReader(data, size+1)
+	} // else we read till EOF.
+
 	// Tee reader combines incoming data stream and md5, data read
 	// from input stream is written to md5.
 	teeReader := io.TeeReader(data, md5Writer)
@@ -343,13 +350,17 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.
 	}
 
 	// Erasure code and write across all disks.
-	newEInfos, n, err := erasureCreateFile(onlineDisks, minioMetaBucket, tempErasureObj, "part.1", teeReader, eInfos, xl.writeQuorum)
+	newEInfos, sizeWritten, err := erasureCreateFile(onlineDisks, minioMetaBucket, tempErasureObj, "part.1", teeReader, eInfos, xl.writeQuorum)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaBucket, tempErasureObj)
 	}
+
+	// For size == -1, perhaps client is sending in chunked encoding
+	// set the size as size that was actually written.
 	if size == -1 {
-		size = n
+		size = sizeWritten
 	}
+
 	// Save additional erasureMetadata.
 	modTime := time.Now().UTC()