Offload listing to posix layer (#7611)

This PR adds one API WalkCh which
sorts and sends list over the network

Each disk walks independently in a sorted manner.

cmd/merge-walk-pool.go

@@ -0,0 +1,143 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"sync"
+	"time"
+)
+
+const (
+	globalMergeLookupTimeout = time.Minute * 1 // 1 minutes.
+)
+
+// mergeWalk - represents the go routine that does the merge walk.
+type mergeWalk struct {
+	entryChs   []FileInfoCh
+	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
+	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
+}
+
+// MergeWalkPool - pool of mergeWalk go routines.
+// A mergeWalk is added to the pool by Set() and removed either by
+// doing a Release() or if the concerned timer goes off.
+// mergeWalkPool's purpose is to maintain active mergeWalk go-routines in a map so that
+// it can be looked up across related list calls.
+type MergeWalkPool struct {
+	pool    map[listParams][]mergeWalk
+	timeOut time.Duration
+	lock    *sync.Mutex
+}
+
+// NewMergeWalkPool - initialize new tree walk pool.
+func NewMergeWalkPool(timeout time.Duration) *MergeWalkPool {
+	tPool := &MergeWalkPool{
+		pool:    make(map[listParams][]mergeWalk),
+		timeOut: timeout,
+		lock:    &sync.Mutex{},
+	}
+	return tPool
+}
+
+// Release - selects a mergeWalk from the pool based on the input
+// listParams, removes it from the pool, and returns the MergeWalkResult
+// channel.
+// Returns nil if listParams does not have an asccociated mergeWalk.
+func (t MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{}) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	walks, ok := t.pool[params] // Pick the valid walks.
+	if ok {
+		if len(walks) > 0 {
+			// Pop out the first valid walk entry.
+			walk := walks[0]
+			walks = walks[1:]
+			if len(walks) > 0 {
+				t.pool[params] = walks
+			} else {
+				delete(t.pool, params)
+			}
+			walk.endTimerCh <- struct{}{}
+			return walk.entryChs, walk.endWalkCh
+		}
+	}
+	// Release return nil if params not found.
+	return nil, nil
+}
+
+// Set - adds a mergeWalk to the mergeWalkPool.
+// Also starts a timer go-routine that ends when:
+// 1) time.After() expires after t.timeOut seconds.
+//    The expiration is needed so that the mergeWalk go-routine resources are freed after a timeout
+//    if the S3 client does only partial listing of objects.
+// 2) Relase() signals the timer go-routine to end on endTimerCh.
+//    During listing the timer should not timeout and end the mergeWalk go-routine, hence the
+//    timer go-routine should be ended.
+func (t MergeWalkPool) Set(params listParams, resultChs []FileInfoCh, endWalkCh chan struct{}) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	// Should be a buffered channel so that Release() never blocks.
+	endTimerCh := make(chan struct{}, 1)
+
+	walkInfo := mergeWalk{
+		entryChs:   resultChs,
+		endWalkCh:  endWalkCh,
+		endTimerCh: endTimerCh,
+	}
+
+	// Append new walk info.
+	t.pool[params] = append(t.pool[params], walkInfo)
+
+	// Timer go-routine which times out after t.timeOut seconds.
+	go func(endTimerCh <-chan struct{}, walkInfo mergeWalk) {
+		select {
+		// Wait until timeOut
+		case <-time.After(t.timeOut):
+			// Timeout has expired. Remove the mergeWalk from mergeWalkPool and
+			// end the mergeWalk go-routine.
+			t.lock.Lock()
+			walks, ok := t.pool[params]
+			if ok {
+				// Trick of filtering without allocating
+				// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
+				nwalks := walks[:0]
+				// Look for walkInfo, remove it from the walks list.
+				for _, walk := range walks {
+					if !reflect.DeepEqual(walk, walkInfo) {
+						nwalks = append(nwalks, walk)
+					}
+				}
+				if len(nwalks) == 0 {
+					// No more mergeWalk go-routines associated with listParams
+					// hence remove map entry.
+					delete(t.pool, params)
+				} else {
+					// There are more mergeWalk go-routines associated with listParams
+					// hence save the list in the map.
+					t.pool[params] = nwalks
+				}
+			}
+			// Signal the mergeWalk go-routine to die.
+			close(endWalkCh)
+			t.lock.Unlock()
+		case <-endTimerCh:
+			return
+		}
+	}(endTimerCh, walkInfo)
+}