Add PutObjectPart benchmark (#2145)

benchmark-utils_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/hex"
+	"math"
 	"math/rand"
 	"strconv"
 	"testing"
@@ -67,6 +68,90 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.StopTimer()
 }
 
+// Benchmark utility functions for ObjectLayer.PutObjectPart().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark.
+func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	objSize := 128 * 1024 * 1024
+
+	// PutObjectPart returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum, uploadID string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for NewMultipartUpload.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var textPartData []byte
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObjectPart starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize)))
+		for j := 0; j < totalPartsNR; j++ {
+			hasher.Reset()
+			if j < totalPartsNR-1 {
+				textPartData = textData[j*partSize : (j+1)*partSize-1]
+			} else {
+				textPartData = textData[j*partSize:]
+			}
+			hasher.Write([]byte(textPartData))
+			metadata := make(map[string]string)
+			metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+			md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"])
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			}
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
+func benchmarkPutObjectPart(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
+	runBenchMark(b, objLayer)
+}
+
+// closure for returning the put object benchmark executor for given object size in bytes.
+func returnPutObjectPartBenchmark(objSize int) func(*testing.B, ObjectLayer) {
+	// FIXME: Avoid closure.
+	return func(b *testing.B, obj ObjectLayer) {
+		runPutObjectPartBenchmark(b, obj, objSize)
+	}
+}
+
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
 func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
 	// create a temp XL/FS backend.

object-api-getobjectinfo_test.go

@@ -105,6 +105,61 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 }
 
+// Benchmarks for ObjectLayer.PutObjectPart().
+// The intent is to benchamrk PutObjectPart for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB.
+func BenchmarkPutObjectPart5MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObjectPart5MbXL - Benchmark XL.PutObjectPart() for object size of 5MB.
+func BenchmarkPutObjectPart5MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB.
+func BenchmarkPutObjectPart10MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObjectPart10MbXL - Benchmark XL.PutObjectPart() for object size of 10MB.
+func BenchmarkPutObjectPart10MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB.
+func BenchmarkPutObjectPart25MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(25*1024*1024))
+
+}
+
+// BenchmarkPutObjectPart25MbXL - Benchmark XL.PutObjectPart() for object size of 25MB.
+func BenchmarkPutObjectPart25MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(25*1024*1024))
+}
+
+// BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB.
+func BenchmarkPutObjectPart50MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObjectPart50MbXL - Benchmark XL.PutObjectPart() for object size of 50MB.
+func BenchmarkPutObjectPart50MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObjectPart100MbFS - Benchmark FS.PutObjectPart() for object size of 100MB.
+func BenchmarkPutObjectPart100MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObjectPart100MbXL - Benchmark XL.PutObjectPart() for object size of 100MB.
+func BenchmarkPutObjectPart100MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(100*1024*1024))
+}
+
 // Benchmarks for ObjectLayer.PutObject().
 // The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB.
 // Also each of these Benchmarks are run both XL and FS backends.

test-utils_test.go

@@ -341,6 +341,12 @@ func randString(n int) string {
 	return string(b)
 }
 
+// generate random object name.
+func getRandomObjectName() string {
+	return randString(16)
+
+}
+
 // generate random bucket name.
 func getRandomBucketName() string {
 	return randString(60)