mirror of
https://github.com/minio/minio.git
synced 2025-11-20 01:50:24 -05:00
xl/ListFiles: return as many objects as requested. (#1383)
* xl/ListFiles: return as many objects as requested and take care of eof (#1361) * xl/ListFiles: fix review comments. * xl/ListFiles: Add windows filepath translation. * xl/ListFiles: Use slashSeparator instead of "/". Remove filepath.FromSlash() as golang-windows takes care of it automatically.
This commit is contained in:
Krishna Srinivas
committed by
Harshavardhana
Harshavardhana
parent
9685f88b84
commit
4333e529e6
124
xl-v1.go
124
xl-v1.go
@@ -20,7 +20,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
slashpath "path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -416,58 +415,99 @@ func (xl XL) ListFiles(volume, prefix, marker string, recursive bool, count int)
|
||||
if isLeaf {
|
||||
// For leaf for now we just point to the first block, make it
|
||||
// dynamic in future based on the availability of storage disks.
|
||||
markerPath = slashpath.Join(marker, "part.0")
|
||||
markerPath = slashpath.Join(marker, metadataFile)
|
||||
}
|
||||
}
|
||||
|
||||
// List files.
|
||||
fsFilesInfo, eof, err = disk.ListFiles(volume, prefix, markerPath, recursive, count)
|
||||
if err != nil {
|
||||
log.WithFields(logrus.Fields{
|
||||
"volume": volume,
|
||||
"prefix": prefix,
|
||||
"marker": markerPath,
|
||||
"recursive": recursive,
|
||||
"count": count,
|
||||
}).Debugf("ListFiles failed with %s", err)
|
||||
return nil, true, err
|
||||
}
|
||||
|
||||
for _, fsFileInfo := range fsFilesInfo {
|
||||
// Skip metadata files.
|
||||
if strings.HasSuffix(fsFileInfo.Name, metadataFile) {
|
||||
continue
|
||||
for {
|
||||
fsFilesInfo, eof, err = disk.ListFiles(volume, prefix, markerPath, recursive, count)
|
||||
if err != nil {
|
||||
log.WithFields(logrus.Fields{
|
||||
"volume": volume,
|
||||
"prefix": prefix,
|
||||
"marker": markerPath,
|
||||
"recursive": recursive,
|
||||
"count": count,
|
||||
}).Debugf("ListFiles failed with %s", err)
|
||||
return nil, true, err
|
||||
}
|
||||
var fileInfo FileInfo
|
||||
var isLeaf bool
|
||||
if fsFileInfo.Mode.IsDir() {
|
||||
isLeaf = xl.isLeafDirectory(volume, fsFileInfo.Name)
|
||||
for _, fsFileInfo := range fsFilesInfo {
|
||||
// Skip metadata files.
|
||||
if strings.HasSuffix(fsFileInfo.Name, metadataFile) {
|
||||
continue
|
||||
}
|
||||
var fileInfo FileInfo
|
||||
var isLeaf bool
|
||||
if fsFileInfo.Mode.IsDir() {
|
||||
isLeaf = xl.isLeafDirectory(volume, fsFileInfo.Name)
|
||||
}
|
||||
if isLeaf || !fsFileInfo.Mode.IsDir() {
|
||||
// Extract the parent of leaf directory or file to get the
|
||||
// actual name.
|
||||
path := slashpath.Dir(fsFileInfo.Name)
|
||||
fileInfo, err = xl.extractFileInfo(volume, path)
|
||||
if err != nil {
|
||||
log.WithFields(logrus.Fields{
|
||||
"volume": volume,
|
||||
"path": path,
|
||||
}).Debugf("extractFileInfo failed with %s", err)
|
||||
// For a leaf directory, if err is FileNotFound then
|
||||
// perhaps has a missing metadata. Ignore it and let
|
||||
// healing finish its job it will become available soon.
|
||||
if err == errFileNotFound {
|
||||
continue
|
||||
}
|
||||
// For any other errors return to the caller.
|
||||
return nil, true, err
|
||||
}
|
||||
} else {
|
||||
fileInfo = fsFileInfo
|
||||
}
|
||||
filesInfo = append(filesInfo, fileInfo)
|
||||
count--
|
||||
if count == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if isLeaf || !fsFileInfo.Mode.IsDir() {
|
||||
// Extract the parent of leaf directory or file to get the
|
||||
// actual name.
|
||||
path := slashpath.Dir(fsFileInfo.Name)
|
||||
fileInfo, err = xl.extractFileInfo(volume, path)
|
||||
lenFsFilesInfo := len(fsFilesInfo)
|
||||
if lenFsFilesInfo != 0 {
|
||||
// markerPath for the next disk.ListFiles() iteration.
|
||||
markerPath = fsFilesInfo[lenFsFilesInfo-1].Name
|
||||
}
|
||||
if count == 0 && recursive && !strings.HasSuffix(markerPath, metadataFile) {
|
||||
// If last entry is not part.json then loop once more to check if we
|
||||
// have reached eof.
|
||||
fsFilesInfo, eof, err = disk.ListFiles(volume, prefix, markerPath, recursive, 1)
|
||||
if err != nil {
|
||||
log.WithFields(logrus.Fields{
|
||||
"volume": volume,
|
||||
"path": path,
|
||||
}).Debugf("extractFileInfo failed with %s", err)
|
||||
// For a leaf directory, if err is FileNotFound then
|
||||
// perhaps has a missing metadata. Ignore it and let
|
||||
// healing finish its job it will become available soon.
|
||||
if err == errFileNotFound {
|
||||
continue
|
||||
}
|
||||
// For any other errors return to the caller.
|
||||
"volume": volume,
|
||||
"prefix": prefix,
|
||||
"marker": markerPath,
|
||||
"recursive": recursive,
|
||||
"count": 1,
|
||||
}).Debugf("ListFiles failed with %s", err)
|
||||
return nil, true, err
|
||||
}
|
||||
} else {
|
||||
fileInfo = fsFileInfo
|
||||
if !eof {
|
||||
// part.N and part.json are always in pairs and hence this
|
||||
// entry has to be part.json. If not better to manually investigate
|
||||
// and fix it.
|
||||
// For the next ListFiles() call we can safely assume that the
|
||||
// marker is "object/part.json"
|
||||
if !strings.HasSuffix(fsFilesInfo[0].Name, metadataFile) {
|
||||
log.WithFields(logrus.Fields{
|
||||
"volume": volume,
|
||||
"prefix": prefix,
|
||||
"fsFileInfo.Name": fsFilesInfo[0].Name,
|
||||
}).Debugf("ListFiles failed with %s, expected %s to be a part.json file.", err, fsFilesInfo[0].Name)
|
||||
return nil, true, errUnexpected
|
||||
}
|
||||
}
|
||||
}
|
||||
if count == 0 || eof {
|
||||
break
|
||||
}
|
||||
filesInfo = append(filesInfo, fileInfo)
|
||||
}
|
||||
sort.Sort(byFileInfoName(filesInfo))
|
||||
return filesInfo, eof, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user