Add compressed file index (#15247)

cmd/erasure-healing.go

@@ -501,6 +501,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
 			partNumber := latestMeta.Parts[partIndex].Number
+			partIdx := latestMeta.Parts[partIndex].Index
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
@@ -550,7 +551,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 
 				partsMetadata[i].DataDir = dstDataDir
-				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize)
+				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partIdx)
 				partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 					PartNumber: partNumber,
 					Algorithm:  checksumAlgo,

cmd/erasure-metadata.go

@@ -237,12 +237,13 @@ func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 }
 
 // AddObjectPart - add a new object part in order.
-func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) {
+func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, idx []byte) {
 	partInfo := ObjectPartInfo{
 		Number:     partNumber,
 		ETag:       partETag,
 		Size:       partSize,
 		ActualSize: actualSize,
+		Index:      idx,
 	}
 
 	// Update part info if it already exists.

cmd/erasure-metadata_test.go

@@ -23,7 +23,7 @@ import (
 	"testing"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 const ActualSize = 1000
@@ -58,7 +58,7 @@ func TestAddObjectPart(t *testing.T) {
 	for _, testCase := range testCases {
 		if testCase.expectedIndex > -1 {
 			partNumString := strconv.Itoa(testCase.partNum)
-			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, nil)
 		}
 
 		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
@@ -91,7 +91,7 @@ func TestObjectPartIndex(t *testing.T) {
 	// Add some parts for testing.
 	for _, testCase := range testCases {
 		partNumString := strconv.Itoa(testCase.partNum)
-		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, nil)
 	}
 
 	// Add failure test case.
@@ -121,7 +121,7 @@ func TestObjectToPartOffset(t *testing.T) {
 	// Total size of all parts is 5,242,899 bytes.
 	for _, partNum := range []int{1, 2, 4, 5, 7} {
 		partNumString := strconv.Itoa(partNum)
-		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, nil)
 	}
 
 	testCases := []struct {
@@ -160,7 +160,7 @@ func TestObjectToPartOffset(t *testing.T) {
 func TestFindFileInfoInQuorum(t *testing.T) {
 	getNFInfo := func(n int, quorum int, t int64, dataDir string) []FileInfo {
 		fi := newFileInfo("test", 8, 8)
-		fi.AddObjectPart(1, "etag", 100, 100)
+		fi.AddObjectPart(1, "etag", 100, 100, nil)
 		fi.ModTime = time.Unix(t, 0)
 		fi.DataDir = dataDir
 		fis := make([]FileInfo, n)

cmd/erasure-multipart.go

@@ -653,9 +653,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	fi.ModTime = UTCNow()
 
 	md5hex := r.MD5CurrentHexString()
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
 
 	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
+	fi.AddObjectPart(partID, md5hex, n, data.ActualSize(), index)
 
 	for i, disk := range onlineDisks {
 		if disk == OfflineDisk {
@@ -947,6 +951,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 

cmd/erasure-object.go

@@ -815,6 +815,10 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	if n < data.Size() {
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
 
 	for i, w := range writers {
 		if w == nil {
@@ -823,7 +827,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1071,6 +1075,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
+	var compIndex []byte
+	if opts.IndexCB != nil {
+		compIndex = opts.IndexCB()
+	}
 	if !opts.NoLock {
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -1091,7 +1099,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), compIndex)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,

cmd/erasure-single-drive.go

@@ -849,6 +849,11 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			// Make sure to avoid writing to disks which we couldn't complete in erasure.Encode()
@@ -856,7 +861,7 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1082,6 +1087,11 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		defer lk.Unlock(lkctx.Cancel)
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1092,7 +1102,7 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -2369,8 +2379,13 @@ func (es *erasureSingle) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	md5hex := r.MD5CurrentHexString()
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
+	fi.AddObjectPart(partID, md5hex, n, data.ActualSize(), index)
 
 	for i, disk := range onlineDisks {
 		if disk == OfflineDisk {
@@ -2668,6 +2683,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 

cmd/object-api-interface.go

@@ -78,6 +78,10 @@ type ObjectOptions struct {
 	WalkAscending bool // return Walk results in ascending order of versions
 
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
+
+	// IndexCB will return any index created but the compression.
+	// Object must have been read at this point.
+	IndexCB func() []byte
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-utils.go

@@ -20,6 +20,8 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/hmac"
+	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -43,12 +45,15 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/trie"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/sio"
 )
 
 const (
@@ -513,12 +518,12 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec {
 // Returns the compressed offset which should be skipped.
 // If encrypted offsets are adjusted for encrypted block headers/trailers.
 // Since de-compression is after decryption encryption overhead is only added to compressedOffset.
-func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64, firstPart int) {
+func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]byte, error)) (compressedOffset int64, partSkip int64, firstPart int, decryptSkip int64, seqNum uint32) {
 	var skipLength int64
 	var cumulativeActualSize int64
 	var firstPartIdx int
-	if len(objectInfo.Parts) > 0 {
+	if len(oi.Parts) > 0 {
-		for i, part := range objectInfo.Parts {
+		for i, part := range oi.Parts {
 			cumulativeActualSize += part.ActualSize
 			if cumulativeActualSize <= offset {
 				compressedOffset += part.Size
@@ -529,8 +534,52 @@ func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset
 			}
 		}
 	}
+	partSkip = offset - skipLength
 
-	return compressedOffset, offset - skipLength, firstPartIdx
+	// Load index and skip more if feasible.
+	if partSkip > 0 && len(oi.Parts) > firstPartIdx && len(oi.Parts[firstPartIdx].Index) > 0 {
+		_, isEncrypted := crypto.IsEncrypted(oi.UserDefined)
+		if isEncrypted {
+			dec, err := decrypt(oi.Parts[firstPartIdx].Index)
+			if err == nil {
+				// Load Index
+				var idx s2.Index
+				_, err := idx.Load(restoreIndexHeaders(dec))
+
+				// Find compressed/uncompressed offsets of our partskip
+				compOff, uCompOff, err2 := idx.Find(partSkip)
+
+				if err == nil && err2 == nil && compOff > 0 {
+					// Encrypted.
+					const sseDAREEncPackageBlockSize = SSEDAREPackageBlockSize + SSEDAREPackageMetaSize
+					// Number of full blocks in skipped area
+					seqNum = uint32(compOff / SSEDAREPackageBlockSize)
+					// Skip this many inside a decrypted block to get to compression block start
+					decryptSkip = compOff % SSEDAREPackageBlockSize
+					// Skip this number of full blocks.
+					skipEnc := compOff / SSEDAREPackageBlockSize
+					skipEnc *= sseDAREEncPackageBlockSize
+					compressedOffset += skipEnc
+					// Skip this number of uncompressed bytes.
+					partSkip -= uCompOff
+				}
+			}
+		} else {
+			// Not encrypted
+			var idx s2.Index
+			_, err := idx.Load(restoreIndexHeaders(oi.Parts[firstPartIdx].Index))
+
+			// Find compressed/uncompressed offsets of our partskip
+			compOff, uCompOff, err2 := idx.Find(partSkip)
+
+			if err == nil && err2 == nil && compOff > 0 {
+				compressedOffset += compOff
+				partSkip -= uCompOff
+			}
+		}
+	}
+
+	return compressedOffset, partSkip, firstPartIdx, decryptSkip, seqNum
 }
 
 // GetObjectReader is a type that wraps a reader with a lock to
@@ -618,6 +667,8 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 		if err != nil {
 			return nil, 0, 0, err
 		}
+		var decryptSkip int64
+		var seqNum uint32
 
 		off, length = int64(0), oi.Size
 		decOff, decLength := int64(0), actualSize
@@ -626,10 +677,14 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 			if err != nil {
 				return nil, 0, 0, err
 			}
-
+			decrypt := func(b []byte) ([]byte, error) {
+				return b, nil
+			}
+			if isEncrypted {
+				decrypt = oi.compressionIndexDecrypt
+			}
 			// In case of range based queries on multiparts, the offset and length are reduced.
-			off, decOff, firstPart = getCompressedOffsets(oi, off)
+			off, decOff, firstPart, decryptSkip, seqNum = getCompressedOffsets(oi, off, decrypt)
-
 			decLength = length
 			length = oi.Size - off
 			// For negative length we read everything.
@@ -646,7 +701,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 			if isEncrypted {
 				copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
 				// Attach decrypter on inputReader
-				inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, firstPart, oi, copySource)
+				inputReader, err = DecryptBlocksRequestR(inputReader, h, seqNum, firstPart, oi, copySource)
 				if err != nil {
 					// Call the cleanup funcs
 					for i := len(cFns) - 1; i >= 0; i-- {
@@ -654,10 +709,18 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 					}
 					return nil, err
 				}
+				if decryptSkip > 0 {
+					inputReader = ioutil.NewSkipReader(inputReader, decryptSkip)
+				}
 				oi.Size = decLength
 			}
 			// Decompression reader.
-			s2Reader := s2.NewReader(inputReader)
+			var dopts []s2.ReaderOption
+			if off > 0 {
+				// We are not starting at the beginning, so ignore stream identifiers.
+				dopts = append(dopts, s2.ReaderIgnoreStreamIdentifier())
+			}
+			s2Reader := s2.NewReader(inputReader, dopts...)
 			// Apply the skipLen and limit on the decompressed stream.
 			if decOff > 0 {
 				if err = s2Reader.Skip(decOff); err != nil {
@@ -778,6 +841,41 @@ func (g *GetObjectReader) Close() error {
 	return nil
 }
 
+func compressionIndexEncrypter(key crypto.ObjectKey, input func() []byte) func() []byte {
+	var data []byte
+	var fetched bool
+	return func() []byte {
+		if !fetched {
+			data = input()
+			fetched = true
+		}
+		if len(data) == 0 {
+			return data
+		}
+		var buffer bytes.Buffer
+		mac := hmac.New(sha256.New, key[:])
+		mac.Write([]byte("compression-index"))
+		if _, err := sio.Encrypt(&buffer, bytes.NewReader(data), sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()}); err != nil {
+			logger.CriticalIf(context.Background(), errors.New("unable to encrypt compression index using object key"))
+		}
+		return buffer.Bytes()
+	}
+}
+
+func (o *ObjectInfo) compressionIndexDecrypt(input []byte) ([]byte, error) {
+	if len(input) == 0 {
+		return input, nil
+	}
+
+	key, err := decryptObjectInfo(nil, o.Bucket, o.Name, o.UserDefined)
+	if err != nil {
+		return nil, err
+	}
+	mac := hmac.New(sha256.New, key)
+	mac.Write([]byte("compression-index"))
+	return sio.DecryptBuffer(nil, input, sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()})
+}
+
 // SealMD5CurrFn seals md5sum with object encryption key and returns sealed
 // md5sum
 type SealMD5CurrFn func([]byte) []byte
@@ -888,11 +986,13 @@ func init() {
 // input 'on' is always recommended such that this function works
 // properly, because we do not wish to create an object even if
 // client closed the stream prematurely.
-func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
+func newS2CompressReader(r io.Reader, on int64) (rc io.ReadCloser, idx func() []byte) {
 	pr, pw := io.Pipe()
 	// Copy input to compressor
+	comp := s2.NewWriter(pw, compressOpts...)
+	indexCh := make(chan []byte, 1)
 	go func() {
-		comp := s2.NewWriter(pw, compressOpts...)
+		defer close(indexCh)
 		cn, err := io.Copy(comp, r)
 		if err != nil {
 			comp.Close()
@@ -907,9 +1007,25 @@ func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
 			return
 		}
 		// Close the stream.
+		// If more than 8MB was written, generate index.
+		if cn > 8<<20 {
+			idx, err := comp.CloseIndex()
+			idx = removeIndexHeaders(idx)
+			indexCh <- idx
+			pw.CloseWithError(err)
+			return
+		}
 		pw.CloseWithError(comp.Close())
 	}()
-	return pr
+	var gotIdx []byte
+	return pr, func() []byte {
+		if gotIdx != nil {
+			return gotIdx
+		}
+		// Will get index or nil if closed.
+		gotIdx = <-indexCh
+		return gotIdx
+	}
 }
 
 // compressSelfTest performs a self-test to ensure that compression
@@ -933,7 +1049,7 @@ func compressSelfTest() {
 		}
 	}
 	const skip = 2<<20 + 511
-	r := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)))
+	r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)))
 	b, err := io.ReadAll(r)
 	failOnErr(err)
 	failOnErr(r.Close())
@@ -1012,3 +1128,65 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
 	wantLeft := uint64(float64(total) * (1.0 - diskFillFraction))
 	return available > wantLeft
 }
+
+// removeIndexHeaders will trim all headers and trailers from a given index.
+// This is expected to save 20 bytes.
+// These can be restored using RestoreIndexHeaders.
+// This removes a layer of security, but is the most compact representation.
+// Returns nil if headers contains errors.
+// The returned slice references the provided slice.
+func removeIndexHeaders(b []byte) []byte {
+	const save = 4 + len(s2.S2IndexHeader) + len(s2.S2IndexTrailer) + 4
+	if len(b) <= save {
+		return nil
+	}
+	if b[0] != s2.ChunkTypeIndex {
+		return nil
+	}
+	chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
+	b = b[4:]
+
+	// Validate we have enough...
+	if len(b) < chunkLen {
+		return nil
+	}
+	b = b[:chunkLen]
+
+	if !bytes.Equal(b[:len(s2.S2IndexHeader)], []byte(s2.S2IndexHeader)) {
+		return nil
+	}
+	b = b[len(s2.S2IndexHeader):]
+	if !bytes.HasSuffix(b, []byte(s2.S2IndexTrailer)) {
+		return nil
+	}
+	b = bytes.TrimSuffix(b, []byte(s2.S2IndexTrailer))
+
+	if len(b) < 4 {
+		return nil
+	}
+	return b[:len(b)-4]
+}
+
+// restoreIndexHeaders will index restore headers removed by RemoveIndexHeaders.
+// No error checking is performed on the input.
+func restoreIndexHeaders(in []byte) []byte {
+	if len(in) == 0 {
+		return nil
+	}
+	b := make([]byte, 0, 4+len(s2.S2IndexHeader)+len(in)+len(s2.S2IndexTrailer)+4)
+	b = append(b, s2.ChunkTypeIndex, 0, 0, 0)
+	b = append(b, []byte(s2.S2IndexHeader)...)
+	b = append(b, in...)
+
+	var tmp [4]byte
+	binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(s2.S2IndexTrailer)))
+	b = append(b, tmp[:4]...)
+	// Trailer
+	b = append(b, []byte(s2.S2IndexTrailer)...)
+
+	chunkLen := len(b) - 4 /*skippableFrameHeader*/
+	b[1] = uint8(chunkLen >> 0)
+	b[2] = uint8(chunkLen >> 8)
+	b[3] = uint8(chunkLen >> 16)
+	return b
+}

cmd/object-api-utils_test.go

@@ -593,7 +593,7 @@ func TestGetCompressedOffsets(t *testing.T) {
 		},
 	}
 	for i, test := range testCases {
-		startOffset, snappyStartOffset, firstPart := getCompressedOffsets(test.objInfo, test.offset)
+		startOffset, snappyStartOffset, firstPart, _, _ := getCompressedOffsets(test.objInfo, test.offset, nil)
 		if startOffset != test.startOffset {
 			t.Errorf("Test %d - expected startOffset %d but received %d",
 				i, test.startOffset, startOffset)
@@ -611,19 +611,20 @@ func TestGetCompressedOffsets(t *testing.T) {
 
 func TestS2CompressReader(t *testing.T) {
 	tests := []struct {
-		name string
+		name    string
-		data []byte
+		data    []byte
+		wantIdx bool
 	}{
 		{name: "empty", data: nil},
-		{name: "small", data: []byte("hello, world")},
+		{name: "small", data: []byte("hello, world!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")},
-		{name: "large", data: bytes.Repeat([]byte("hello, world"), 1000)},
+		{name: "large", data: bytes.Repeat([]byte("hello, world"), 1000000), wantIdx: true},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
 
-			r := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)))
+			r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)))
 			defer r.Close()
 
 			var rdrBuf bytes.Buffer
@@ -631,7 +632,26 @@ func TestS2CompressReader(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-
+			r.Close()
+			idx := idxCB()
+			if !tt.wantIdx && len(idx) > 0 {
+				t.Errorf("index returned above threshold")
+			}
+			if tt.wantIdx {
+				if idx == nil {
+					t.Errorf("no index returned")
+				}
+				var index s2.Index
+				_, err = index.Load(restoreIndexHeaders(idx))
+				if err != nil {
+					t.Errorf("error loading index: %v", err)
+				}
+				t.Log("size:", len(idx))
+				t.Log(string(index.JSON()))
+				if index.TotalUncompressed != int64(len(tt.data)) {
+					t.Errorf("Expected size %d, got %d", len(tt.data), index.TotalUncompressed)
+				}
+			}
 			var stdBuf bytes.Buffer
 			w := s2.NewWriter(&stdBuf)
 			_, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf)

cmd/object-handlers.go

@@ -1164,7 +1164,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 
 		reader = etag.NewReader(reader, nil)
-		s2c := newS2CompressReader(reader, actualSize)
+		s2c, cb := newS2CompressReader(reader, actualSize)
+		dstOpts.IndexCB = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, reader)
 		length = -1
@@ -1308,6 +1309,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
+				if dstOpts.IndexCB != nil {
+					dstOpts.IndexCB = compressionIndexEncrypter(objEncKey, dstOpts.IndexCB)
+				}
 			}
 		}
 	}
@@ -1715,6 +1719,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	})
 
 	actualSize := size
+	var idxCb func() []byte
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
 		// Storing the compression metadata.
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
@@ -1727,8 +1732,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		// Set compression metrics.
-		s2c := newS2CompressReader(actualReader, actualSize)
+		var s2c io.ReadCloser
+		s2c, idxCb = newS2CompressReader(actualReader, actualSize)
 		defer s2c.Close()
+
 		reader = etag.Wrap(s2c, actualReader)
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
@@ -1751,6 +1758,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	opts.IndexCB = idxCb
 
 	if api.CacheAPI() != nil {
 		putObject = api.CacheAPI().PutObject
@@ -1813,6 +1821,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
+			if opts.IndexCB != nil {
+				opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB)
+			}
 		}
 	}
 
@@ -2061,6 +2072,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		}
 
 		actualSize := size
+		var idxCb func() []byte
 		if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
 			// Storing the compression metadata.
 			metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
@@ -2072,8 +2084,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			}
 
 			// Set compression metrics.
-			s2c := newS2CompressReader(actualReader, actualSize)
+			s2c, cb := newS2CompressReader(actualReader, actualSize)
 			defer s2c.Close()
+			idxCb = cb
 			reader = etag.Wrap(s2c, actualReader)
 			size = -1 // Since compressed size is un-predictable.
 		}
@@ -2100,6 +2113,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return err
 		}
 		opts.MTime = info.ModTime()
+		opts.IndexCB = idxCb
 
 		retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 		if s3err == ErrNone && retentionMode.Valid() {
@@ -2153,6 +2167,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 					return err
 				}
 			}
+			if opts.IndexCB != nil {
+				opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB)
+			}
 		}
 
 		// Ensure that metadata does not contain sensitive information
@@ -2571,8 +2588,10 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 	// Compress only if the compression is enabled during initial multipart.
+	var idxCb func() []byte
 	if isCompressed {
-		s2c := newS2CompressReader(reader, actualPartSize)
+		s2c, cb := newS2CompressReader(reader, actualPartSize)
+		idxCb = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, reader)
 		length = -1
@@ -2589,6 +2608,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	dstOpts.IndexCB = idxCb
 
 	rawReader := srcInfo.Reader
 	pReader := NewPutObjReader(rawReader)
@@ -2643,6 +2663,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if dstOpts.IndexCB != nil {
+			dstOpts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, dstOpts.IndexCB)
+		}
 	}
 
 	srcInfo.PutObjReader = pReader
@@ -2821,6 +2844,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 
+	var idxCb func() []byte
 	if objectAPI.IsCompressionSupported() && isCompressed {
 		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
@@ -2829,7 +2853,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		}
 
 		// Set compression metrics.
-		s2c := newS2CompressReader(actualReader, actualSize)
+		s2c, cb := newS2CompressReader(actualReader, actualSize)
+		idxCb = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, actualReader)
 		size = -1   // Since compressed size is un-predictable.
@@ -2904,7 +2929,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if idxCb != nil {
+			idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
+		}
 	}
+	opts.IndexCB = idxCb
 
 	putObjectPart := objectAPI.PutObjectPart
 	if api.CacheAPI() != nil {

cmd/xl-storage-format-v1.go

@@ -132,6 +132,7 @@ type ObjectPartInfo struct {
 	Number     int    `json:"number"`
 	Size       int64  `json:"size"`
 	ActualSize int64  `json:"actualSize"`
+	Index      []byte `json:"index,omitempty" msg:"index,omitempty"`
 }
 
 // ChecksumInfo - carries checksums of individual scattered parts per disk.

cmd/xl-storage-format-v1_gen.go

@@ -593,6 +593,12 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ActualSize")
 				return
 			}
+		case "index":
+			z.Index, err = dc.ReadBytes(z.Index)
+			if err != nil {
+				err = msgp.WrapError(err, "Index")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -606,9 +612,23 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 4
+	// omitempty: check for empty values
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
+	if z.Index == nil {
+		zb0001Len--
+		zb0001Mask |= 0x10
+	}
+	// variable map header, size zb0001Len
+	err = en.Append(0x80 | uint8(zb0001Len))
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "ETag"
-	err = en.Append(0x84, 0xa4, 0x45, 0x54, 0x61, 0x67)
+	err = en.Append(0xa4, 0x45, 0x54, 0x61, 0x67)
 	if err != nil {
 		return
 	}
@@ -647,15 +667,38 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ActualSize")
 		return
 	}
+	if (zb0001Mask & 0x10) == 0 { // if not empty
+		// write "index"
+		err = en.Append(0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
+		if err != nil {
+			return
+		}
+		err = en.WriteBytes(z.Index)
+		if err != nil {
+			err = msgp.WrapError(err, "Index")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
+	// omitempty: check for empty values
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
+	if z.Index == nil {
+		zb0001Len--
+		zb0001Mask |= 0x10
+	}
+	// variable map header, size zb0001Len
+	o = append(o, 0x80|uint8(zb0001Len))
+	if zb0001Len == 0 {
+		return
+	}
 	// string "ETag"
-	o = append(o, 0x84, 0xa4, 0x45, 0x54, 0x61, 0x67)
+	o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67)
 	o = msgp.AppendString(o, z.ETag)
 	// string "Number"
 	o = append(o, 0xa6, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
@@ -666,6 +709,11 @@ func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ActualSize"
 	o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.ActualSize)
+	if (zb0001Mask & 0x10) == 0 { // if not empty
+		// string "index"
+		o = append(o, 0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
+		o = msgp.AppendBytes(o, z.Index)
+	}
 	return
 }
 
@@ -711,6 +759,12 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ActualSize")
 				return
 			}
+		case "index":
+			z.Index, bts, err = msgp.ReadBytesBytes(bts, z.Index)
+			if err != nil {
+				err = msgp.WrapError(err, "Index")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -725,7 +779,7 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *ObjectPartInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.ETag) + 7 + msgp.IntSize + 5 + msgp.Int64Size + 11 + msgp.Int64Size
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.ETag) + 7 + msgp.IntSize + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 6 + msgp.BytesPrefixSize + len(z.Index)
 	return
 }
 

cmd/xl-storage-format-v2.go

@@ -167,6 +167,7 @@ type xlMetaV2Object struct {
 	PartETags          []string          `json:"PartETags" msg:"PartETags,allownil"`             // Part ETags
 	PartSizes          []int64           `json:"PartSizes" msg:"PartSizes"`                      // Part Sizes
 	PartActualSizes    []int64           `json:"PartASizes,omitempty" msg:"PartASizes,allownil"` // Part ActualSizes (compression)
+	PartIndices        [][]byte          `json:"PartIndices,omitempty" msg:"PartIdx,omitempty"`  // Part Indexes (compression)
 	Size               int64             `json:"Size" msg:"Size"`                                // Object version size
 	ModTime            int64             `json:"MTime" msg:"MTime"`                              // Object version modified time
 	MetaSys            map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,allownil"`       // Object version internal metadata
@@ -574,6 +575,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 			fi.Parts[i].ETag = j.PartETags[i]
 		}
 		fi.Parts[i].ActualSize = j.PartActualSizes[i]
+		if len(j.PartIndices) > 0 {
+			fi.Parts[i].Index = j.PartIndices[i]
+		}
 	}
 	fi.Erasure.Checksums = make([]ChecksumInfo, len(j.PartSizes))
 	for i := range fi.Parts {
@@ -1471,6 +1475,13 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 				break
 			}
 		}
+		for i := range fi.Parts {
+			// Only add indices if any.
+			if len(fi.Parts[i].Index) > 0 {
+				ventry.ObjectV2.PartIndices = make([][]byte, len(fi.Parts))
+				break
+			}
+		}
 		for i := range fi.Erasure.Distribution {
 			ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i])
 		}
@@ -1482,6 +1493,9 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 			}
 			ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number
 			ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize
+			if len(ventry.ObjectV2.PartIndices) > 0 {
+				ventry.ObjectV2.PartIndices[i] = fi.Parts[i].Index
+			}
 		}
 
 		tierFVIDKey := ReservedMetadataPrefixLower + tierFVID

cmd/xl-storage-format-v2_gen.go

@@ -935,6 +935,25 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "PartIdx":
+			var zb0009 uint32
+			zb0009, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices")
+				return
+			}
+			if cap(z.PartIndices) >= int(zb0009) {
+				z.PartIndices = (z.PartIndices)[:zb0009]
+			} else {
+				z.PartIndices = make([][]byte, zb0009)
+			}
+			for za0008 := range z.PartIndices {
+				z.PartIndices[za0008], err = dc.ReadBytes(z.PartIndices[za0008])
+				if err != nil {
+					err = msgp.WrapError(err, "PartIndices", za0008)
+					return
+				}
+			}
 		case "Size":
 			z.Size, err = dc.ReadInt64()
 			if err != nil {
@@ -956,34 +975,34 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.MetaSys = nil
 			} else {
-				var zb0009 uint32
+				var zb0010 uint32
-				zb0009, err = dc.ReadMapHeader()
+				zb0010, err = dc.ReadMapHeader()
 				if err != nil {
 					err = msgp.WrapError(err, "MetaSys")
 					return
 				}
 				if z.MetaSys == nil {
-					z.MetaSys = make(map[string][]byte, zb0009)
+					z.MetaSys = make(map[string][]byte, zb0010)
 				} else if len(z.MetaSys) > 0 {
 					for key := range z.MetaSys {
 						delete(z.MetaSys, key)
 					}
 				}
-				for zb0009 > 0 {
+				for zb0010 > 0 {
-					zb0009--
+					zb0010--
-					var za0008 string
+					var za0009 string
-					var za0009 []byte
+					var za0010 []byte
-					za0008, err = dc.ReadString()
+					za0009, err = dc.ReadString()
 					if err != nil {
 						err = msgp.WrapError(err, "MetaSys")
 						return
 					}
-					za0009, err = dc.ReadBytes(za0009)
+					za0010, err = dc.ReadBytes(za0010)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaSys", za0008)
+						err = msgp.WrapError(err, "MetaSys", za0009)
 						return
 					}
-					z.MetaSys[za0008] = za0009
+					z.MetaSys[za0009] = za0010
 				}
 			}
 		case "MetaUsr":
@@ -995,34 +1014,34 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.MetaUser = nil
 			} else {
-				var zb0010 uint32
+				var zb0011 uint32
-				zb0010, err = dc.ReadMapHeader()
+				zb0011, err = dc.ReadMapHeader()
 				if err != nil {
 					err = msgp.WrapError(err, "MetaUser")
 					return
 				}
 				if z.MetaUser == nil {
-					z.MetaUser = make(map[string]string, zb0010)
+					z.MetaUser = make(map[string]string, zb0011)
 				} else if len(z.MetaUser) > 0 {
 					for key := range z.MetaUser {
 						delete(z.MetaUser, key)
 					}
 				}
-				for zb0010 > 0 {
+				for zb0011 > 0 {
-					zb0010--
+					zb0011--
-					var za0010 string
 					var za0011 string
-					za0010, err = dc.ReadString()
+					var za0012 string
+					za0011, err = dc.ReadString()
 					if err != nil {
 						err = msgp.WrapError(err, "MetaUser")
 						return
 					}
-					za0011, err = dc.ReadString()
+					za0012, err = dc.ReadString()
 					if err != nil {
-						err = msgp.WrapError(err, "MetaUser", za0010)
+						err = msgp.WrapError(err, "MetaUser", za0011)
 						return
 					}
-					z.MetaUser[za0010] = za0011
+					z.MetaUser[za0011] = za0012
 				}
 			}
 		default:
@@ -1038,9 +1057,23 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 17
+	// omitempty: check for empty values
+	zb0001Len := uint32(18)
+	var zb0001Mask uint32 /* 18 bits */
+	if z.PartIndices == nil {
+		zb0001Len--
+		zb0001Mask |= 0x2000
+	}
+	// variable map header, size zb0001Len
+	err = en.WriteMapHeader(zb0001Len)
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
+	err = en.Append(0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1218,6 +1251,25 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
+	if (zb0001Mask & 0x2000) == 0 { // if not empty
+		// write "PartIdx"
+		err = en.Append(0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
+		if err != nil {
+			return
+		}
+		err = en.WriteArrayHeader(uint32(len(z.PartIndices)))
+		if err != nil {
+			err = msgp.WrapError(err, "PartIndices")
+			return
+		}
+		for za0008 := range z.PartIndices {
+			err = en.WriteBytes(z.PartIndices[za0008])
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices", za0008)
+				return
+			}
+		}
+	}
 	// write "Size"
 	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
 	if err != nil {
@@ -1254,15 +1306,15 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "MetaSys")
 			return
 		}
-		for za0008, za0009 := range z.MetaSys {
+		for za0009, za0010 := range z.MetaSys {
-			err = en.WriteString(za0008)
+			err = en.WriteString(za0009)
 			if err != nil {
 				err = msgp.WrapError(err, "MetaSys")
 				return
 			}
-			err = en.WriteBytes(za0009)
+			err = en.WriteBytes(za0010)
 			if err != nil {
-				err = msgp.WrapError(err, "MetaSys", za0008)
+				err = msgp.WrapError(err, "MetaSys", za0009)
 				return
 			}
 		}
@@ -1283,15 +1335,15 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "MetaUser")
 			return
 		}
-		for za0010, za0011 := range z.MetaUser {
+		for za0011, za0012 := range z.MetaUser {
-			err = en.WriteString(za0010)
+			err = en.WriteString(za0011)
 			if err != nil {
 				err = msgp.WrapError(err, "MetaUser")
 				return
 			}
-			err = en.WriteString(za0011)
+			err = en.WriteString(za0012)
 			if err != nil {
-				err = msgp.WrapError(err, "MetaUser", za0010)
+				err = msgp.WrapError(err, "MetaUser", za0011)
 				return
 			}
 		}
@@ -1302,9 +1354,20 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 17
+	// omitempty: check for empty values
+	zb0001Len := uint32(18)
+	var zb0001Mask uint32 /* 18 bits */
+	if z.PartIndices == nil {
+		zb0001Len--
+		zb0001Mask |= 0x2000
+	}
+	// variable map header, size zb0001Len
+	o = msgp.AppendMapHeader(o, zb0001Len)
+	if zb0001Len == 0 {
+		return
+	}
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
+	o = append(o, 0xa2, 0x49, 0x44)
 	o = msgp.AppendBytes(o, (z.VersionID)[:])
 	// string "DDir"
 	o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72)
@@ -1365,6 +1428,14 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 			o = msgp.AppendInt64(o, z.PartActualSizes[za0007])
 		}
 	}
+	if (zb0001Mask & 0x2000) == 0 { // if not empty
+		// string "PartIdx"
+		o = append(o, 0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
+		o = msgp.AppendArrayHeader(o, uint32(len(z.PartIndices)))
+		for za0008 := range z.PartIndices {
+			o = msgp.AppendBytes(o, z.PartIndices[za0008])
+		}
+	}
 	// string "Size"
 	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.Size)
@@ -1377,9 +1448,9 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 		o = msgp.AppendNil(o)
 	} else {
 		o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys)))
-		for za0008, za0009 := range z.MetaSys {
+		for za0009, za0010 := range z.MetaSys {
-			o = msgp.AppendString(o, za0008)
+			o = msgp.AppendString(o, za0009)
-			o = msgp.AppendBytes(o, za0009)
+			o = msgp.AppendBytes(o, za0010)
 		}
 	}
 	// string "MetaUsr"
@@ -1388,9 +1459,9 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 		o = msgp.AppendNil(o)
 	} else {
 		o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser)))
-		for za0010, za0011 := range z.MetaUser {
+		for za0011, za0012 := range z.MetaUser {
-			o = msgp.AppendString(o, za0010)
 			o = msgp.AppendString(o, za0011)
+			o = msgp.AppendString(o, za0012)
 		}
 	}
 	return
@@ -1575,6 +1646,25 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					}
 				}
 			}
+		case "PartIdx":
+			var zb0009 uint32
+			zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices")
+				return
+			}
+			if cap(z.PartIndices) >= int(zb0009) {
+				z.PartIndices = (z.PartIndices)[:zb0009]
+			} else {
+				z.PartIndices = make([][]byte, zb0009)
+			}
+			for za0008 := range z.PartIndices {
+				z.PartIndices[za0008], bts, err = msgp.ReadBytesBytes(bts, z.PartIndices[za0008])
+				if err != nil {
+					err = msgp.WrapError(err, "PartIndices", za0008)
+					return
+				}
+			}
 		case "Size":
 			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
@@ -1592,34 +1682,34 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				bts = bts[1:]
 				z.MetaSys = nil
 			} else {
-				var zb0009 uint32
+				var zb0010 uint32
-				zb0009, bts, err = msgp.ReadMapHeaderBytes(bts)
+				zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "MetaSys")
 					return
 				}
 				if z.MetaSys == nil {
-					z.MetaSys = make(map[string][]byte, zb0009)
+					z.MetaSys = make(map[string][]byte, zb0010)
 				} else if len(z.MetaSys) > 0 {
 					for key := range z.MetaSys {
 						delete(z.MetaSys, key)
 					}
 				}
-				for zb0009 > 0 {
+				for zb0010 > 0 {
-					var za0008 string
+					var za0009 string
-					var za0009 []byte
+					var za0010 []byte
-					zb0009--
+					zb0010--
-					za0008, bts, err = msgp.ReadStringBytes(bts)
+					za0009, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
 						err = msgp.WrapError(err, "MetaSys")
 						return
 					}
-					za0009, bts, err = msgp.ReadBytesBytes(bts, za0009)
+					za0010, bts, err = msgp.ReadBytesBytes(bts, za0010)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaSys", za0008)
+						err = msgp.WrapError(err, "MetaSys", za0009)
 						return
 					}
-					z.MetaSys[za0008] = za0009
+					z.MetaSys[za0009] = za0010
 				}
 			}
 		case "MetaUsr":
@@ -1627,34 +1717,34 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				bts = bts[1:]
 				z.MetaUser = nil
 			} else {
-				var zb0010 uint32
+				var zb0011 uint32
-				zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
+				zb0011, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "MetaUser")
 					return
 				}
 				if z.MetaUser == nil {
-					z.MetaUser = make(map[string]string, zb0010)
+					z.MetaUser = make(map[string]string, zb0011)
 				} else if len(z.MetaUser) > 0 {
 					for key := range z.MetaUser {
 						delete(z.MetaUser, key)
 					}
 				}
-				for zb0010 > 0 {
+				for zb0011 > 0 {
-					var za0010 string
 					var za0011 string
-					zb0010--
+					var za0012 string
-					za0010, bts, err = msgp.ReadStringBytes(bts)
+					zb0011--
+					za0011, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
 						err = msgp.WrapError(err, "MetaUser")
 						return
 					}
-					za0011, bts, err = msgp.ReadStringBytes(bts)
+					za0012, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaUser", za0010)
+						err = msgp.WrapError(err, "MetaUser", za0011)
 						return
 					}
-					z.MetaUser[za0010] = za0011
+					z.MetaUser[za0011] = za0012
 				}
 			}
 		default:
@@ -1675,18 +1765,22 @@ func (z *xlMetaV2Object) Msgsize() (s int) {
 	for za0005 := range z.PartETags {
 		s += msgp.StringPrefixSize + len(z.PartETags[za0005])
 	}
-	s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
+	s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 8 + msgp.ArrayHeaderSize
+	for za0008 := range z.PartIndices {
+		s += msgp.BytesPrefixSize + len(z.PartIndices[za0008])
+	}
+	s += 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
 	if z.MetaSys != nil {
-		for za0008, za0009 := range z.MetaSys {
+		for za0009, za0010 := range z.MetaSys {
-			_ = za0009
+			_ = za0010
-			s += msgp.StringPrefixSize + len(za0008) + msgp.BytesPrefixSize + len(za0009)
+			s += msgp.StringPrefixSize + len(za0009) + msgp.BytesPrefixSize + len(za0010)
 		}
 	}
 	s += 8 + msgp.MapHeaderSize
 	if z.MetaUser != nil {
-		for za0010, za0011 := range z.MetaUser {
+		for za0011, za0012 := range z.MetaUser {
-			_ = za0011
+			_ = za0012
-			s += msgp.StringPrefixSize + len(za0010) + msgp.StringPrefixSize + len(za0011)
+			s += msgp.StringPrefixSize + len(za0011) + msgp.StringPrefixSize + len(za0012)
 		}
 	}
 	return