diff --git a/benchmark-utils_test.go b/benchmark-utils_test.go index 728e380c3..faebffcec 100644 --- a/benchmark-utils_test.go +++ b/benchmark-utils_test.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" "encoding/hex" + "io/ioutil" "math" "math/rand" "strconv" @@ -261,3 +262,123 @@ func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) { runGetObjectBenchmark(b, obj, objSize) } } + +// Parallel benchmark utility functions for ObjectLayer.PutObject(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. +func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + // create bucket. + err = obj.MakeBucket(bucket) + if err != nil { + b.Fatal(err) + } + + // PutObject returns md5Sum of the object inserted. + // md5Sum variable is assigned with that value. + var md5Sum string + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for PutObject. + hasher := md5.New() + hasher.Write([]byte(textData)) + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + // insert the object. + md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + if err != nil { + b.Fatal(err) + } + if md5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", md5Sum, metadata["md5Sum"]) + } + i++ + } + }) + + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// closure for returning the put object benchmark executor for given object size in bytes. +func returnPutObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) { + // FIXME: Avoid closure. + return func(b *testing.B, obj ObjectLayer) { + runPutObjectBenchmarkParallel(b, obj, objSize) + } +} + +// Parallel benchmark utility functions for ObjectLayer.GetObject(). +// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark. +func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + // create bucket. + err = obj.MakeBucket(bucket) + if err != nil { + b.Fatal(err) + } + + // PutObject returns md5Sum of the object inserted. + // md5Sum variable is assigned with that value. + var md5Sum string + for i := 0; i < 10; i++ { + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for PutObject. + // PutObject is the functions which writes the data onto the FS/XL backend. + hasher := md5.New() + hasher.Write([]byte(textData)) + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + // insert the object. + md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + if err != nil { + b.Fatal(err) + } + if md5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + } + } + + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for GetObject starts here. Reset the benchmark timer. + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard) + if err != nil { + b.Error(err) + } + i++ + if i == 10 { + i = 0 + } + } + }) + // Benchmark ends here. Stop timer. + b.StopTimer() + +} + +// closure for returning the get object benchmark executor for given object size in bytes. +// FIXME: Avoid closure. +func returnGetObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) { + return func(b *testing.B, obj ObjectLayer) { + runGetObjectBenchmarkParallel(b, obj, objSize) + } +} diff --git a/object-api-getobject_test.go b/object-api-getobject_test.go index 54e8c9f88..c84e20394 100644 --- a/object-api-getobject_test.go +++ b/object-api-getobject_test.go @@ -298,3 +298,127 @@ func BenchmarkGetObject1GbFS(b *testing.B) { func BenchmarkGetObject1GbXL(b *testing.B) { benchmarkGetObject(b, "XL", returnGetObjectBenchmark(1024*1024*1024)) } + +// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB. +// Also each of these BenchmarkParallels are run both XL and FS backends. + +// BenchmarkParallelGetObjectVerySmallFS - BenchmarkParallel FS.GetObject() for object size of 10 bytes. +func BenchmarkParallelGetObjectVerySmallFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10)) +} + +// BenchmarkParallelGetObjectVerySmallXL - BenchmarkParallel XL.GetObject() for object size of 10 bytes. +func BenchmarkParallelGetObjectVerySmallXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10)) +} + +// BenchmarkParallelGetObject10KbFS - BenchmarkParallel FS.GetObject() for object size of 10KB. +func BenchmarkParallelGetObject10KbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024)) +} + +// BenchmarkParallelGetObject10KbXL - BenchmarkParallel XL.GetObject() for object size of 10KB. +func BenchmarkParallelGetObject10KbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024)) +} + +// BenchmarkParallelGetObject100KbFS - BenchmarkParallel FS.GetObject() for object size of 100KB. +func BenchmarkParallelGetObject100KbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024)) +} + +// BenchmarkParallelGetObject100KbXL - BenchmarkParallel XL.GetObject() for object size of 100KB. +func BenchmarkParallelGetObject100KbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024)) +} + +// BenchmarkParallelGetObject1MbFS - BenchmarkParallel FS.GetObject() for object size of 1MB. +func BenchmarkParallelGetObject1MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024)) +} + +// BenchmarkParallelGetObject1MbXL - BenchmarkParallel XL.GetObject() for object size of 1MB. +func BenchmarkParallelGetObject1MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024)) +} + +// BenchmarkParallelGetObject5MbFS - BenchmarkParallel FS.GetObject() for object size of 5MB. +func BenchmarkParallelGetObject5MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(5*1024*1024)) +} + +// BenchmarkParallelGetObject5MbXL - BenchmarkParallel XL.GetObject() for object size of 5MB. +func BenchmarkParallelGetObject5MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(5*1024*1024)) +} + +// BenchmarkParallelGetObject10MbFS - BenchmarkParallel FS.GetObject() for object size of 10MB. +func BenchmarkParallelGetObject10MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024*1024)) +} + +// BenchmarkParallelGetObject10MbXL - BenchmarkParallel XL.GetObject() for object size of 10MB. +func BenchmarkParallelGetObject10MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024*1024)) +} + +// BenchmarkParallelGetObject25MbFS - BenchmarkParallel FS.GetObject() for object size of 25MB. +func BenchmarkParallelGetObject25MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(25*1024*1024)) + +} + +// BenchmarkParallelGetObject25MbXL - BenchmarkParallel XL.GetObject() for object size of 25MB. +func BenchmarkParallelGetObject25MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(25*1024*1024)) +} + +// BenchmarkParallelGetObject50MbFS - BenchmarkParallel FS.GetObject() for object size of 50MB. +func BenchmarkParallelGetObject50MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(50*1024*1024)) +} + +// BenchmarkParallelGetObject50MbXL - BenchmarkParallel XL.GetObject() for object size of 50MB. +func BenchmarkParallelGetObject50MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(50*1024*1024)) +} + +// BenchmarkParallelGetObject100MbFS - BenchmarkParallel FS.GetObject() for object size of 100MB. +func BenchmarkParallelGetObject100MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024*1024)) +} + +// BenchmarkParallelGetObject100MbXL - BenchmarkParallel XL.GetObject() for object size of 100MB. +func BenchmarkParallelGetObject100MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024*1024)) +} + +// BenchmarkParallelGetObject200MbFS - BenchmarkParallel FS.GetObject() for object size of 200MB. +func BenchmarkParallelGetObject200MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(200*1024*1024)) +} + +// BenchmarkParallelGetObject200MbXL - BenchmarkParallel XL.GetObject() for object size of 200MB. +func BenchmarkParallelGetObject200MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(200*1024*1024)) +} + +// BenchmarkParallelGetObject500MbFS - BenchmarkParallel FS.GetObject() for object size of 500MB. +func BenchmarkParallelGetObject500MbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(500*1024*1024)) +} + +// BenchmarkParallelGetObject500MbXL - BenchmarkParallel XL.GetObject() for object size of 500MB. +func BenchmarkParallelGetObject500MbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(500*1024*1024)) +} + +// BenchmarkParallelGetObject1GbFS - BenchmarkParallel FS.GetObject() for object size of 1GB. +func BenchmarkParallelGetObject1GbFS(b *testing.B) { + benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024*1024)) +} + +// BenchmarkParallelGetObjectGbXL - BenchmarkParallel XL.GetObject() for object size of 1GB. +func BenchmarkParallelGetObject1GbXL(b *testing.B) { + benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024*1024)) +} diff --git a/object-api-putobject_test.go b/object-api-putobject_test.go index c0e541ed0..4f7fcd691 100644 --- a/object-api-putobject_test.go +++ b/object-api-putobject_test.go @@ -469,3 +469,123 @@ func BenchmarkPutObject1GbFS(b *testing.B) { func BenchmarkPutObject1GbXL(b *testing.B) { benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024)) } + +func BenchmarkParallelPutObjectVerySmallFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10)) +} + +// BenchmarkParallelPutObjectVerySmallXL - BenchmarkParallel XL.PutObject() for object size of 10 bytes. +func BenchmarkParallelPutObjectVerySmallXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10)) +} + +// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB. +func BenchmarkParallelPutObject10KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024)) +} + +// BenchmarkParallelPutObject10KbXL - BenchmarkParallel XL.PutObject() for object size of 10KB. +func BenchmarkParallelPutObject10KbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024)) +} + +// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB. +func BenchmarkParallelPutObject100KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024)) +} + +// BenchmarkParallelPutObject100KbXL - BenchmarkParallel XL.PutObject() for object size of 100KB. +func BenchmarkParallelPutObject100KbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024)) +} + +// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB. +func BenchmarkParallelPutObject1MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024)) +} + +// BenchmarkParallelPutObject1MbXL - BenchmarkParallel XL.PutObject() for object size of 1MB. +func BenchmarkParallelPutObject1MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024)) +} + +// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB. +func BenchmarkParallelPutObject5MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(5*1024*1024)) +} + +// BenchmarkParallelPutObject5MbXL - BenchmarkParallel XL.PutObject() for object size of 5MB. +func BenchmarkParallelPutObject5MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(5*1024*1024)) +} + +// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB. +func BenchmarkParallelPutObject10MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024*1024)) +} + +// BenchmarkParallelPutObject10MbXL - BenchmarkParallel XL.PutObject() for object size of 10MB. +func BenchmarkParallelPutObject10MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024*1024)) +} + +// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB. +func BenchmarkParallelPutObject25MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(25*1024*1024)) + +} + +// BenchmarkParallelPutObject25MbXL - BenchmarkParallel XL.PutObject() for object size of 25MB. +func BenchmarkParallelPutObject25MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(25*1024*1024)) +} + +// BenchmarkParallelPutObject50MbFS - BenchmarkParallel FS.PutObject() for object size of 50MB. +func BenchmarkParallelPutObject50MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(50*1024*1024)) +} + +// BenchmarkParallelPutObject50MbXL - BenchmarkParallel XL.PutObject() for object size of 50MB. +func BenchmarkParallelPutObject50MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(50*1024*1024)) +} + +// BenchmarkParallelPutObject100MbFS - BenchmarkParallel FS.PutObject() for object size of 100MB. +func BenchmarkParallelPutObject100MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024*1024)) +} + +// BenchmarkParallelPutObject100MbXL - BenchmarkParallel XL.PutObject() for object size of 100MB. +func BenchmarkParallelPutObject100MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024*1024)) +} + +// BenchmarkParallelPutObject200MbFS - BenchmarkParallel FS.PutObject() for object size of 200MB. +func BenchmarkParallelPutObject200MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(200*1024*1024)) +} + +// BenchmarkParallelPutObject200MbXL - BenchmarkParallel XL.PutObject() for object size of 200MB. +func BenchmarkParallelPutObject200MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(200*1024*1024)) +} + +// BenchmarkParallelPutObject500MbFS - BenchmarkParallel FS.PutObject() for object size of 500MB. +func BenchmarkParallelPutObject500MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(500*1024*1024)) +} + +// BenchmarkParallelPutObject500MbXL - BenchmarkParallel XL.PutObject() for object size of 500MB. +func BenchmarkParallelPutObject500MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(500*1024*1024)) +} + +// BenchmarkParallelPutObject1GbFS - BenchmarkParallel FS.PutObject() for object size of 1GB. +func BenchmarkParallelPutObject1GbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024*1024)) +} + +// BenchmarkParallelPutObjectGbXL - BenchmarkParallel XL.PutObject() for object size of 1GB. +func BenchmarkParallelPutObject1GbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024*1024)) +}