Benchmarks for ObjectLayer.PutObject() (#2029)

This commit is contained in:
karthic rao 2016-06-29 10:42:36 +05:30 committed by Harshavardhana
parent 748dc80047
commit 59366d8f4c
2 changed files with 202 additions and 10 deletions

View File

@ -26,6 +26,69 @@ import (
"time"
)
// Benchmark utility functions for ObjectLayer.PutObject().
// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
var err error
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucket(bucket)
if err != nil {
b.Fatal(err)
}
// PutObject returns md5Sum of the object inserted.
// md5Sum variable is assigned with that value.
var md5Sum string
// get text data generated for number of bytes equal to object size.
textData := generateBytesData(objSize)
// generate md5sum for the generated data.
// md5sum of the data to written is required as input for PutObject.
hasher := md5.New()
hasher.Write([]byte(textData))
metadata := make(map[string]string)
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
b.ReportAllocs()
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
b.ResetTimer()
for i := 0; i < b.N; i++ {
// insert the object.
md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
if err != nil {
b.Fatal(err)
}
if md5Sum != metadata["md5Sum"] {
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
}
}
// Benchmark ends here. Stop timer.
b.StopTimer()
}
// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
// create a temp XL/FS backend.
objLayer, disks, err := makeTestBackend(instanceType)
if err != nil {
b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
}
// cleaning up the backend by removing all the directories and files created on function return.
defer removeRoots(disks)
// calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
runBenchMark(b, objLayer)
}
// closure for returning the put object benchmark executor for given object size in bytes.
func returnPutObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
// FIXME: Avoid closure.
return func(b *testing.B, obj ObjectLayer) {
runPutObjectBenchmark(b, obj, objSize)
}
}
// Benchmark utility functions for ObjectLayer.GetObject().
// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
var err error
@ -75,9 +138,9 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
b.StopTimer()
}
func generateBytesData(size int) []byte {
// randomly picks a character and returns its equivalent byte array.
getRandomByte := func() []byte {
// randomly picks a character and returns its equivalent byte array.
func getRandomByte() []byte {
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
// seeding the random number generator.
rand.Seed(time.Now().UnixNano())
@ -85,7 +148,10 @@ func generateBytesData(size int) []byte {
// pick a character randomly.
b = letterBytes[rand.Intn(len(letterBytes))]
return []byte{b}
}
}
// picks a random byte and repeats it to size bytes.
func generateBytesData(size int) []byte {
// repeat the random character chosen size
return bytes.Repeat(getRandomByte(), size)
}
@ -104,6 +170,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, runBenchMark func(b *
}
// closure for returning the get object benchmark executor for given object size in bytes.
// FIXME: Avoid closure.
func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
return func(b *testing.B, obj ObjectLayer) {
runGetObjectBenchmark(b, obj, objSize)

View File

@ -105,6 +105,131 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
}
}
// Benchmarks for ObjectLayer.PutObject().
// The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB.
// Also each of these Benchmarks are run both XL and FS backends.
// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10))
}
// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes.
func BenchmarkPutObjectVerySmallXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10))
}
// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024))
}
// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB.
func BenchmarkPutObject10KbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024))
}
// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024))
}
// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB.
func BenchmarkPutObject100KbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024))
}
// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024))
}
// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB.
func BenchmarkPutObject1MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024))
}
// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(5*1024*1024))
}
// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB.
func BenchmarkPutObject5MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(5*1024*1024))
}
// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024*1024))
}
// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB.
func BenchmarkPutObject10MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024*1024))
}
// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(25*1024*1024))
}
// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB.
func BenchmarkPutObject25MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(25*1024*1024))
}
// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(50*1024*1024))
}
// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB.
func BenchmarkPutObject50MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(50*1024*1024))
}
// BenchmarkPutObject100MbFS - Benchmark FS.PutObject() for object size of 100MB.
func BenchmarkPutObject100MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024*1024))
}
// BenchmarkPutObject100MbXL - Benchmark XL.PutObject() for object size of 100MB.
func BenchmarkPutObject100MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024*1024))
}
// BenchmarkPutObject200MbFS - Benchmark FS.PutObject() for object size of 200MB.
func BenchmarkPutObject200MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(200*1024*1024))
}
// BenchmarkPutObject200MbXL - Benchmark XL.PutObject() for object size of 200MB.
func BenchmarkPutObject200MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(200*1024*1024))
}
// BenchmarkPutObject500MbFS - Benchmark FS.PutObject() for object size of 500MB.
func BenchmarkPutObject500MbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(500*1024*1024))
}
// BenchmarkPutObject500MbXL - Benchmark XL.PutObject() for object size of 500MB.
func BenchmarkPutObject500MbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(500*1024*1024))
}
// BenchmarkPutObject1GbFS - Benchmark FS.PutObject() for object size of 1GB.
func BenchmarkPutObject1GbFS(b *testing.B) {
benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024*1024))
}
// BenchmarkPutObjectGbXL - Benchmark XL.PutObject() for object size of 1GB.
func BenchmarkPutObject1GbXL(b *testing.B) {
benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024))
}
// Benchmarks for ObjectLayer.GetObject().
// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
// Also each of these Benchmarks are run both XL and FS backends.