api: refactor list object handling in fs backend

When list object is invoked, it creates a goroutine if not available
for given parameters else uses existing goroutine.  These goroutines
are alive for 15 seconds for further continuation list object request
else they exit.

Fixes #1076

pkg/fs/dir.go

@@ -0,0 +1,245 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	// listObjectsLimit - maximum list objects limit
+	listObjectsLimit = 1000
+)
+
+// IsDirEmpty - returns whether given directory is empty or not
+func IsDirEmpty(dirname string) (status bool, err error) {
+	f, err := os.Open(dirname)
+	if err == nil {
+		defer f.Close()
+		if _, err = f.Readdirnames(1); err == io.EOF {
+			status = true
+			err = nil
+		}
+	}
+
+	return
+}
+
+// IsDirExist - returns whether given directory is exist or not
+func IsDirExist(dirname string) (status bool, err error) {
+	fi, err := os.Lstat(dirname)
+	if err == nil {
+		status = fi.IsDir()
+	}
+
+	return
+}
+
+// byName implements sort.Interface for sorting os.FileInfo list
+type byName []os.FileInfo
+
+func (f byName) Len() int      { return len(f) }
+func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
+func (f byName) Less(i, j int) bool {
+	n1 := f[i].Name()
+	if f[i].IsDir() {
+		n1 = n1 + string(os.PathSeparator)
+	}
+
+	n2 := f[j].Name()
+	if f[j].IsDir() {
+		n2 = n2 + string(os.PathSeparator)
+	}
+
+	return n1 < n2
+}
+
+// ObjectInfo - object info
+type ObjectInfo struct {
+	Name         string
+	ModifiedTime time.Time
+	Checksum     string
+	Size         int64
+	IsDir        bool
+	Err          error
+}
+
+// readDir - read 'scanDir' directory.  It returns list of ObjectInfo where
+//           each object name is appended with 'namePrefix'
+func readDir(scanDir, namePrefix string) (objInfos []ObjectInfo) {
+	f, err := os.Open(scanDir)
+	if err != nil {
+		objInfos = append(objInfos, ObjectInfo{Err: err})
+		return
+	}
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		f.Close()
+		objInfos = append(objInfos, ObjectInfo{Err: err})
+		return
+	}
+
+	f.Close()
+	sort.Sort(byName(fis))
+
+	// make []ObjectInfo from []FileInfo
+	for _, fi := range fis {
+		name := fi.Name()
+		if namePrefix != "" {
+			name = namePrefix + "/" + name
+		}
+
+		if fi.IsDir() {
+			name += "/"
+		}
+
+		objInfos = append(objInfos, ObjectInfo{
+			Name:         name,
+			ModifiedTime: fi.ModTime(),
+			Checksum:     "",
+			Size:         fi.Size(),
+			IsDir:        fi.IsDir(),
+		})
+	}
+
+	return
+}
+
+// ObjectInfoChannel - object info channel
+type ObjectInfoChannel struct {
+	ch        <-chan ObjectInfo
+	objInfo   *ObjectInfo
+	closed    bool
+	timeoutCh <-chan struct{}
+	timedOut  bool
+}
+
+func (oic *ObjectInfoChannel) Read() (ObjectInfo, bool) {
+	if oic.closed {
+		return ObjectInfo{}, false
+	}
+
+	if oic.objInfo == nil {
+		// first read
+		if oi, ok := <-oic.ch; ok {
+			oic.objInfo = &oi
+		} else {
+			oic.closed = true
+			return ObjectInfo{}, false
+		}
+	}
+
+	retObjInfo := *oic.objInfo
+	status := true
+	oic.objInfo = nil
+
+	// read once more to know whether it was last read
+	if oi, ok := <-oic.ch; ok {
+		oic.objInfo = &oi
+	} else {
+		oic.closed = true
+	}
+
+	return retObjInfo, status
+}
+
+// IsClosed - return whether channel is closed or not
+func (oic ObjectInfoChannel) IsClosed() bool {
+	if oic.objInfo != nil {
+		return false
+	}
+
+	return oic.closed
+}
+
+// IsTimedOut - return whether channel is closed due to timeout
+func (oic ObjectInfoChannel) IsTimedOut() bool {
+	if oic.timedOut {
+		return true
+	}
+
+	select {
+	case _, ok := <-oic.timeoutCh:
+		if ok {
+			oic.timedOut = true
+			return true
+		}
+		return false
+	default:
+		return false
+	}
+}
+
+// treeWalk - walk into 'scanDir' recursively when 'recursive' is true.
+//            It uses 'bucketDir' to get name prefix for object name.
+func treeWalk(scanDir, bucketDir string, recursive bool) ObjectInfoChannel {
+	objectInfoCh := make(chan ObjectInfo, listObjectsLimit)
+	timeoutCh := make(chan struct{}, 1)
+
+	// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel
+	go func() {
+		defer close(objectInfoCh)
+		defer close(timeoutCh)
+
+		// send function - returns true if ObjectInfo is sent
+		//                 within (time.Second * 15) else false on time-out
+		send := func(oi ObjectInfo) bool {
+			timer := time.After(time.Second * 15)
+			select {
+			case objectInfoCh <- oi:
+				return true
+			case <-timer:
+				timeoutCh <- struct{}{}
+				return false
+			}
+		}
+
+		namePrefix := strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
+		if strings.HasPrefix(namePrefix, "/") {
+			/* remove beginning "/" */
+			namePrefix = namePrefix[1:]
+		}
+
+		for objInfos := readDir(scanDir, namePrefix); len(objInfos) > 0; {
+			var objInfo ObjectInfo
+			objInfo, objInfos = objInfos[0], objInfos[1:]
+			if !send(objInfo) {
+				return
+			}
+
+			if objInfo.IsDir && recursive {
+				scanDir := filepath.Join(bucketDir, filepath.FromSlash(objInfo.Name))
+
+				namePrefix = strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
+				if strings.HasPrefix(namePrefix, "/") {
+					/* remove beginning "/" */
+					namePrefix = namePrefix[1:]
+				}
+
+				objInfos = append(readDir(scanDir, namePrefix), objInfos...)
+			}
+		}
+	}()
+
+	return ObjectInfoChannel{ch: objectInfoCh, timeoutCh: timeoutCh}
+}