caching: Optimize memory allocations. (#3405)

This change brings in changes at multiple places

 - Reuse buffers at almost all locations ranging
   from rpc, fs, xl, checksum etc.
 - Change caching behavior to disable itself
   under low memory conditions i.e < 8GB of RAM.
 - Only objects cached are of size 1/10th the size
   of the cache for example if 4GB is the cache size
   the maximum object size which will be cached
   is going to be 400MB. This change is an
   optimization to cache more objects rather
   than few larger objects.
 - If object cache is enabled default GC
   percent has been reduced to 20% in lieu
   with newly found behavior of GC. If the cache
   utilization reaches 75% of the maximum value
   GC percent is reduced to 10% to make GC
   more aggressive.
 - Do not use *bytes.Buffer* due to its growth
   requirements. For every allocation *bytes.Buffer*
   allocates an additional buffer for its internal
   purposes. This is undesirable for us, so
   implemented a new cappedWriter which is capped to a
   desired size, beyond this all writes rejected.

Possible fix for #3403.

cmd/xl-v1.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"runtime/debug"
 	"sort"
 	"strings"
 	"sync"
@@ -42,8 +43,9 @@ const (
 	// Uploads metadata file carries per multipart object metadata.
 	uploadsJSONFile = "uploads.json"
 
-	// 8GiB cache by default.
-	maxCacheSize = 8 * humanize.GiByte
+	// Represents the minimum required RAM size before
+	// we enable caching.
+	minRAMSize = 8 * humanize.GiByte
 
 	// Maximum erasure blocks.
 	maxErasureBlocks = 16
@@ -92,9 +94,6 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
 	// Calculate data and parity blocks.
 	dataBlocks, parityBlocks := len(newStorageDisks)/2, len(newStorageDisks)/2
 
-	// Initialize object cache.
-	objCache := objcache.New(globalMaxCacheSize, globalCacheExpiry)
-
 	// Initialize list pool.
 	listPool := newTreeWalkPool(globalLookupTimeout)
 
@@ -103,13 +102,25 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
 
 	// Initialize xl objects.
 	xl := &xlObjects{
-		mutex:           &sync.Mutex{},
-		storageDisks:    newStorageDisks,
-		dataBlocks:      dataBlocks,
-		parityBlocks:    parityBlocks,
-		listPool:        listPool,
-		objCache:        objCache,
-		objCacheEnabled: !objCacheDisabled,
+		mutex:        &sync.Mutex{},
+		storageDisks: newStorageDisks,
+		dataBlocks:   dataBlocks,
+		parityBlocks: parityBlocks,
+		listPool:     listPool,
+	}
+
+	// Object cache is enabled when _MINIO_CACHE env is missing.
+	// and cache size is > 0.
+	xl.objCacheEnabled = !objCacheDisabled && globalMaxCacheSize > 0
+
+	// Check if object cache is enabled.
+	if xl.objCacheEnabled {
+		// Initialize object cache.
+		objCache := objcache.New(globalMaxCacheSize, globalCacheExpiry)
+		objCache.OnEviction = func(key string) {
+			debug.FreeOSMemory()
+		}
+		xl.objCache = objCache
 	}
 
 	// Initialize meta volume, if volume already exists ignores it.