From 59366d8f4c3f981b17d9fceeb05a54b801721ac5 Mon Sep 17 00:00:00 2001 From: karthic rao Date: Wed, 29 Jun 2016 10:42:36 +0530 Subject: [PATCH] Benchmarks for ObjectLayer.PutObject() (#2029) --- benchmark-utils_test.go | 87 ++++++++++++++++++--- object-api-getobjectinfo_test.go | 125 +++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 10 deletions(-) diff --git a/benchmark-utils_test.go b/benchmark-utils_test.go index c9afc30a5..3faa60cb8 100644 --- a/benchmark-utils_test.go +++ b/benchmark-utils_test.go @@ -26,6 +26,69 @@ import ( "time" ) +// Benchmark utility functions for ObjectLayer.PutObject(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. +func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + // create bucket. + err = obj.MakeBucket(bucket) + if err != nil { + b.Fatal(err) + } + + // PutObject returns md5Sum of the object inserted. + // md5Sum variable is assigned with that value. + var md5Sum string + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for PutObject. + hasher := md5.New() + hasher.Write([]byte(textData)) + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + b.ResetTimer() + for i := 0; i < b.N; i++ { + // insert the object. + md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata) + if err != nil { + b.Fatal(err) + } + if md5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + } + } + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function. +func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) { + // create a temp XL/FS backend. + objLayer, disks, err := makeTestBackend(instanceType) + if err != nil { + b.Fatalf("Failed obtaining Temp Backend: %s", err) + } + // cleaning up the backend by removing all the directories and files created on function return. + defer removeRoots(disks) + // calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark. + runBenchMark(b, objLayer) +} + +// closure for returning the put object benchmark executor for given object size in bytes. +func returnPutObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) { + // FIXME: Avoid closure. + return func(b *testing.B, obj ObjectLayer) { + runPutObjectBenchmark(b, obj, objSize) + } +} + +// Benchmark utility functions for ObjectLayer.GetObject(). // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark. func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { var err error @@ -75,17 +138,20 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.StopTimer() } + +// randomly picks a character and returns its equivalent byte array. +func getRandomByte() []byte { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + // seeding the random number generator. + rand.Seed(time.Now().UnixNano()) + var b byte + // pick a character randomly. + b = letterBytes[rand.Intn(len(letterBytes))] + return []byte{b} +} + +// picks a random byte and repeats it to size bytes. func generateBytesData(size int) []byte { - // randomly picks a character and returns its equivalent byte array. - getRandomByte := func() []byte { - const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - // seeding the random number generator. - rand.Seed(time.Now().UnixNano()) - var b byte - // pick a character randomly. - b = letterBytes[rand.Intn(len(letterBytes))] - return []byte{b} - } // repeat the random character chosen size return bytes.Repeat(getRandomByte(), size) } @@ -104,6 +170,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, runBenchMark func(b * } // closure for returning the get object benchmark executor for given object size in bytes. +// FIXME: Avoid closure. func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) { return func(b *testing.B, obj ObjectLayer) { runGetObjectBenchmark(b, obj, objSize) diff --git a/object-api-getobjectinfo_test.go b/object-api-getobjectinfo_test.go index de54bb1db..cc7e6d39f 100644 --- a/object-api-getobjectinfo_test.go +++ b/object-api-getobjectinfo_test.go @@ -105,6 +105,131 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) { } } +// Benchmarks for ObjectLayer.PutObject(). +// The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB. +// Also each of these Benchmarks are run both XL and FS backends. + +// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes. +func BenchmarkPutObjectVerySmallFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10)) +} + +// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes. +func BenchmarkPutObjectVerySmallXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10)) +} + +// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB. +func BenchmarkPutObject10KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024)) +} + +// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB. +func BenchmarkPutObject10KbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024)) +} + +// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB. +func BenchmarkPutObject100KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024)) +} + +// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB. +func BenchmarkPutObject100KbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024)) +} + +// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB. +func BenchmarkPutObject1MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024)) +} + +// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB. +func BenchmarkPutObject1MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024)) +} + +// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB. +func BenchmarkPutObject5MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(5*1024*1024)) +} + +// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB. +func BenchmarkPutObject5MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(5*1024*1024)) +} + +// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB. +func BenchmarkPutObject10MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024*1024)) +} + +// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB. +func BenchmarkPutObject10MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024*1024)) +} + +// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB. +func BenchmarkPutObject25MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(25*1024*1024)) + +} + +// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB. +func BenchmarkPutObject25MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(25*1024*1024)) +} + +// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB. +func BenchmarkPutObject50MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(50*1024*1024)) +} + +// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB. +func BenchmarkPutObject50MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(50*1024*1024)) +} + +// BenchmarkPutObject100MbFS - Benchmark FS.PutObject() for object size of 100MB. +func BenchmarkPutObject100MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024*1024)) +} + +// BenchmarkPutObject100MbXL - Benchmark XL.PutObject() for object size of 100MB. +func BenchmarkPutObject100MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024*1024)) +} + +// BenchmarkPutObject200MbFS - Benchmark FS.PutObject() for object size of 200MB. +func BenchmarkPutObject200MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(200*1024*1024)) +} + +// BenchmarkPutObject200MbXL - Benchmark XL.PutObject() for object size of 200MB. +func BenchmarkPutObject200MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(200*1024*1024)) +} + +// BenchmarkPutObject500MbFS - Benchmark FS.PutObject() for object size of 500MB. +func BenchmarkPutObject500MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(500*1024*1024)) +} + +// BenchmarkPutObject500MbXL - Benchmark XL.PutObject() for object size of 500MB. +func BenchmarkPutObject500MbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(500*1024*1024)) +} + +// BenchmarkPutObject1GbFS - Benchmark FS.PutObject() for object size of 1GB. +func BenchmarkPutObject1GbFS(b *testing.B) { + benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024*1024)) +} + +// BenchmarkPutObjectGbXL - Benchmark XL.PutObject() for object size of 1GB. +func BenchmarkPutObject1GbXL(b *testing.B) { + benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024)) +} + // Benchmarks for ObjectLayer.GetObject(). // The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB. // Also each of these Benchmarks are run both XL and FS backends.