Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111
This commit is contained in:
Harshavardhana
2020-06-12 20:04:01 -07:00
committed by GitHub
parent 43d6e3ae06
commit 4915433bd2
203 changed files with 13833 additions and 6919 deletions

View File

@@ -20,7 +20,6 @@ package cmd
import (
"os"
"strings"
"syscall"
)
@@ -32,43 +31,24 @@ func readDir(dirPath string) (entries []string, err error) {
// readDir applies the filter function on each entries at dirPath, doesn't recurse into
// the directory itself.
func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
d, err := os.Open(dirPath)
f, err := os.Open(dirPath)
if err != nil {
// File is really not found.
if os.IsNotExist(err) {
return errFileNotFound
}
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return errFileNotFound
}
return err
}
defer d.Close()
st, err := d.Stat()
if err != nil {
return err
}
// Not a directory return error.
if !st.IsDir() {
return errFileAccessDenied
return osErrToFileErr(err)
}
defer f.Close()
data := &syscall.Win32finddata{}
for {
e := syscall.FindNextFile(syscall.Handle(d.Fd()), data)
e := syscall.FindNextFile(syscall.Handle(f.Fd()), data)
if e != nil {
if e == syscall.ERROR_NO_MORE_FILES {
break
} else {
err = &os.PathError{
return osErrToFileErr(&os.PathError{
Op: "FindNextFile",
Path: dirPath,
Err: e,
}
return err
})
}
}
name := syscall.UTF16ToString(data.FileName[0:])
@@ -82,7 +62,7 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
if data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
typ = os.ModeDir
}
if err = filter(name, typ); err == errDoneForNow {
if e = filter(name, typ); e == errDoneForNow {
// filtering requested to return by caller.
return nil
}
@@ -93,43 +73,25 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
// Return N entries at the directory dirPath. If count is -1, return all entries
func readDirN(dirPath string, count int) (entries []string, err error) {
d, err := os.Open(dirPath)
f, err := os.Open(dirPath)
if err != nil {
// File is really not found.
if os.IsNotExist(err) {
return nil, errFileNotFound
}
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return nil, errFileNotFound
}
return nil, err
}
defer d.Close()
st, err := d.Stat()
if err != nil {
return nil, err
}
// Not a directory return error.
if !st.IsDir() {
return nil, errFileNotFound
return nil, osErrToFileErr(err)
}
defer f.Close()
data := &syscall.Win32finddata{}
for count != 0 {
err = syscall.FindNextFile(syscall.Handle(d.Fd()), data)
if err != nil {
if err == syscall.ERROR_NO_MORE_FILES {
e := syscall.FindNextFile(syscall.Handle(f.Fd()), data)
if e != nil {
if e == syscall.ERROR_NO_MORE_FILES {
break
} else {
return nil, &os.PathError{
return nil, osErrToFileErr(&os.PathError{
Op: "FindNextFile",
Path: dirPath,
Err: err,
}
Err: e,
})
}
}
@@ -147,5 +109,10 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
}
count--
}
return entries, nil
}
func globalSync() {
// no-op on windows
}