Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/naughty-disk_test.go

@@ -142,18 +142,25 @@ func (d *naughtyDisk) WalkSplunk(volume, path, marker string, endWalkCh <-chan s
 	return d.disk.WalkSplunk(volume, path, marker, endWalkCh)
 }
 
-func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (chan FileInfo, error) {
+func (d *naughtyDisk) WalkVersions(volume, path, marker string, recursive bool, endWalkVersionsCh <-chan struct{}) (chan FileInfoVersions, error) {
 	if err := d.calcError(); err != nil {
 		return nil, err
 	}
-	return d.disk.Walk(volume, path, marker, recursive, leafFile, readMetadataFn, endWalkCh)
+	return d.disk.WalkVersions(volume, path, marker, recursive, endWalkVersionsCh)
 }
 
-func (d *naughtyDisk) ListDir(volume, path string, count int, leafFile string) (entries []string, err error) {
+func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfo, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.Walk(volume, path, marker, recursive, endWalkCh)
+}
+
+func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) {
 	if err := d.calcError(); err != nil {
 		return []string{}, err
 	}
-	return d.disk.ListDir(volume, path, count, leafFile)
+	return d.disk.ListDir(volume, path, count)
 }
 
 func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
@@ -184,6 +191,13 @@ func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error {
 	return d.disk.AppendFile(volume, path, buf)
 }
 
+func (d *naughtyDisk) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath string) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath)
+}
+
 func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
 	if err := d.calcError(); err != nil {
 		return err
@@ -191,11 +205,18 @@ func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string)
 	return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
 }
 
-func (d *naughtyDisk) StatFile(volume string, path string) (file FileInfo, err error) {
+func (d *naughtyDisk) CheckParts(volume string, path string, fi FileInfo) (err error) {
 	if err := d.calcError(); err != nil {
-		return FileInfo{}, err
+		return err
 	}
-	return d.disk.StatFile(volume, path)
+	return d.disk.CheckParts(volume, path, fi)
+}
+
+func (d *naughtyDisk) CheckFile(volume string, path string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.CheckFile(volume, path)
 }
 
 func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
@@ -205,19 +226,36 @@ func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
 	return d.disk.DeleteFile(volume, path)
 }
 
-func (d *naughtyDisk) DeleteFileBulk(volume string, paths []string) ([]error, error) {
-	errs := make([]error, len(paths))
-	for idx, path := range paths {
-		errs[idx] = d.disk.DeleteFile(volume, path)
+func (d *naughtyDisk) DeleteVersions(volume string, versions []FileInfo) []error {
+	if err := d.calcError(); err != nil {
+		errs := make([]error, len(versions))
+		for i := range errs {
+			errs[i] = err
+		}
+		return errs
 	}
-	return errs, nil
+	return d.disk.DeleteVersions(volume, versions)
 }
 
-func (d *naughtyDisk) DeletePrefixes(volume string, paths []string) ([]error, error) {
+func (d *naughtyDisk) WriteMetadata(volume, path string, fi FileInfo) (err error) {
 	if err := d.calcError(); err != nil {
-		return nil, err
+		return err
 	}
-	return d.disk.DeletePrefixes(volume, paths)
+	return d.disk.WriteMetadata(volume, path, fi)
+}
+
+func (d *naughtyDisk) DeleteVersion(volume string, path string, fi FileInfo) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteVersion(volume, path, fi)
+}
+
+func (d *naughtyDisk) ReadVersion(volume string, path string, versionID string) (fi FileInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return FileInfo{}, err
+	}
+	return d.disk.ReadVersion(volume, path, versionID)
 }
 
 func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
@@ -234,9 +272,9 @@ func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error
 	return d.disk.ReadAll(volume, path)
 }
 
-func (d *naughtyDisk) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (d *naughtyDisk) VerifyFile(volume, path string, fi FileInfo) error {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.VerifyFile(volume, path, size, algo, sum, shardSize)
+	return d.disk.VerifyFile(volume, path, fi)
 }