Inline small file data in xl.meta file (#11758)

cmd/xl-storage-format-v2.go

@@ -18,12 +18,15 @@ package cmd
 
 import (
 	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/tinylib/msgp/msgp"
+
 	"github.com/google/uuid"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -33,28 +36,55 @@ var (
 	// XL header specifies the format
 	xlHeader = [4]byte{'X', 'L', '2', ' '}
 
-	// XLv2 version 1
-	xlVersionV1 = [4]byte{'1', ' ', ' ', ' '}
+	// Current version being written.
+	xlVersionCurrent [4]byte
 )
 
-func checkXL2V1(buf []byte) error {
+const (
+	// Breaking changes.
+	// Newer versions cannot be read by older software.
+	// This will prevent downgrades to incompatible versions.
+	xlVersionMajor = 1
+
+	// Non breaking changes.
+	// Bumping this is informational, but should be done
+	// if any change is made to the data stored, bumping this
+	// will allow to detect the exact version later.
+	xlVersionMinor = 1
+)
+
+func init() {
+	binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor)
+	binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor)
+}
+
+// checkXL2V1 will check if the metadata has correct header and is a known major version.
+// The remaining payload and versions are returned.
+func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) {
 	if len(buf) <= 8 {
-		return fmt.Errorf("xlMeta: no data")
+		return payload, 0, 0, fmt.Errorf("xlMeta: no data")
 	}
 
 	if !bytes.Equal(buf[:4], xlHeader[:]) {
-		return fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4])
+		return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4])
 	}
 
-	if !bytes.Equal(buf[4:8], xlVersionV1[:]) {
-		return fmt.Errorf("xlMeta: unknown XLv2 version, expected %v, got %v", xlVersionV1[:4], buf[4:8])
+	if bytes.Equal(buf[4:8], []byte("1   ")) {
+		// Set as 1,0.
+		major, minor = 1, 0
+	} else {
+		major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8])
+	}
+	if major > xlVersionMajor {
+		return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major)
 	}
 
-	return nil
+	return buf[8:], major, minor, nil
 }
 
 func isXL2V1Format(buf []byte) bool {
-	return checkXL2V1(buf) == nil
+	_, _, _, err := checkXL2V1(buf)
+	return err == nil
 }
 
 // The []journal contains all the different versions of the object.
@@ -199,6 +229,317 @@ func (j xlMetaV2Version) Valid() bool {
 // the journals for the object.
 type xlMetaV2 struct {
 	Versions []xlMetaV2Version `json:"Versions" msg:"Versions"`
+
+	// data will contain raw data if any.
+	// data will be one or more versions indexed by storage dir.
+	// To remove all data set to nil.
+	data xlMetaInlineData `msg:"-"`
+}
+
+// xlMetaInlineData is serialized data in [string][]byte pairs.
+//
+//msgp:ignore xlMetaInlineData
+type xlMetaInlineData []byte
+
+// xlMetaInlineDataVer indicates the vesrion of the inline data structure.
+const xlMetaInlineDataVer = 1
+
+// versionOK returns whether the version is ok.
+func (x xlMetaInlineData) versionOK() bool {
+	if len(x) == 0 {
+		return true
+	}
+	return x[0] > 0 && x[0] <= xlMetaInlineDataVer
+}
+
+// afterVersion returns the payload after the version, if any.
+func (x xlMetaInlineData) afterVersion() []byte {
+	if len(x) == 0 {
+		return x
+	}
+	return x[1:]
+}
+
+// find the data with key s.
+// Returns nil if not for or an error occurs.
+func (x xlMetaInlineData) find(key string) []byte {
+	if len(x) == 0 || !x.versionOK() {
+		return nil
+	}
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil || sz == 0 {
+		return nil
+	}
+	for i := uint32(0); i < sz; i++ {
+		var found []byte
+		found, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil || sz == 0 {
+			return nil
+		}
+		if string(found) == key {
+			val, _, _ := msgp.ReadBytesZC(buf)
+			return val
+		}
+		// Skip it
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return nil
+		}
+	}
+	return nil
+}
+
+// validate checks if the data is valid.
+// It does not check integrity of the stored data.
+func (x xlMetaInlineData) validate() error {
+	if len(x) == 0 {
+		return nil
+	}
+	if !x.versionOK() {
+		return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0])
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil {
+		return err
+	}
+	for i := uint32(0); i < sz; i++ {
+		var key []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			return err
+		}
+		if len(key) == 0 {
+			return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
+		}
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// validate checks if the data is valid.
+// It does not check integrity of the stored data.
+func (x xlMetaInlineData) list() ([]string, error) {
+	if len(x) == 0 {
+		return nil, nil
+	}
+	if !x.versionOK() {
+		return nil, errors.New("xlMetaInlineData: unknown version")
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil {
+		return nil, err
+	}
+	keys := make([]string, 0, sz)
+	for i := uint32(0); i < sz; i++ {
+		var key []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			return keys, err
+		}
+		if len(key) == 0 {
+			return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
+		}
+		keys = append(keys, string(key))
+		// Skip data...
+		_, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return keys, err
+		}
+	}
+	return keys, nil
+}
+
+func (x xlMetaInlineData) entries() int {
+	if len(x) == 0 || !x.versionOK() {
+		return 0
+	}
+	sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion())
+	return int(sz)
+}
+
+// replace will add or replace a key/value pair.
+func (x *xlMetaInlineData) replace(key string, value []byte) {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	keys := make([][]byte, 0, sz+1)
+	vals := make([][]byte, 0, sz+1)
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	replaced := false
+	for i := uint32(0); i < sz; i++ {
+		var found, foundVal []byte
+		var err error
+		found, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+		keys = append(keys, found)
+		if string(found) == key {
+			vals = append(vals, value)
+			plSize += len(value)
+			replaced = true
+		} else {
+			vals = append(vals, foundVal)
+			plSize += len(foundVal)
+		}
+	}
+	// Add one more.
+	if !replaced {
+		keys = append(keys, []byte(key))
+		vals = append(vals, value)
+		plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+	}
+
+	// Reserialize...
+	payload := make([]byte, 1, plSize)
+	payload[0] = xlMetaInlineDataVer
+	payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
+	for i := range keys {
+		payload = msgp.AppendStringFromBytes(payload, keys[i])
+		payload = msgp.AppendBytes(payload, vals[i])
+	}
+	*x = payload
+	if err := x.validate(); err != nil {
+		panic(err)
+	}
+}
+
+// rename will rename a key.
+// Returns whether the key was found.
+func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	keys := make([][]byte, 0, sz)
+	vals := make([][]byte, 0, sz)
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	found := false
+	for i := uint32(0); i < sz; i++ {
+		var foundKey, foundVal []byte
+		var err error
+		foundKey, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize
+		vals = append(vals, foundVal)
+		if string(foundKey) != oldKey {
+			keys = append(keys, foundKey)
+			plSize += len(foundKey)
+		} else {
+			keys = append(keys, []byte(newKey))
+			plSize += len(newKey)
+			found = true
+		}
+	}
+	// If not found, just return.
+	if !found {
+		return false
+	}
+
+	// Reserialize...
+	payload := make([]byte, 1, plSize)
+	payload[0] = xlMetaInlineDataVer
+	payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
+	for i := range keys {
+		payload = msgp.AppendStringFromBytes(payload, keys[i])
+		payload = msgp.AppendBytes(payload, vals[i])
+	}
+	*x = payload
+	return true
+}
+
+// remove will remove a key.
+// Returns whether the key was found.
+func (x *xlMetaInlineData) remove(key string) bool {
+	in := x.afterVersion()
+	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
+	keys := make([][]byte, 0, sz)
+	vals := make([][]byte, 0, sz)
+
+	// Version plus header...
+	plSize := 1 + msgp.MapHeaderSize
+	found := false
+	for i := uint32(0); i < sz; i++ {
+		var foundKey, foundVal []byte
+		var err error
+		foundKey, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			break
+		}
+		foundVal, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			break
+		}
+		if string(foundKey) != key {
+			plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
+			keys = append(keys, foundKey)
+			vals = append(vals, foundVal)
+		} else {
+			found = true
+		}
+	}
+	// If not found, just return.
+	if !found {
+		return false
+	}
+	// If none left...
+	if len(keys) == 0 {
+		*x = nil
+		return true
+	}
+
+	// Reserialize...
+	payload := make([]byte, 1, plSize)
+	payload[0] = xlMetaInlineDataVer
+	payload = msgp.AppendMapHeader(payload, uint32(len(keys)))
+	for i := range keys {
+		payload = msgp.AppendStringFromBytes(payload, keys[i])
+		payload = msgp.AppendBytes(payload, vals[i])
+	}
+	*x = payload
+	return true
+}
+
+// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it.
+// If any error occurs the unmodified data is returned.
+func xlMetaV2TrimData(buf []byte) []byte {
+	metaBuf, min, maj, err := checkXL2V1(buf)
+	if err != nil {
+		return buf
+	}
+	if maj == 1 && min < 1 {
+		// First version to carry data.
+		return buf
+	}
+	// Skip header
+	_, metaBuf, err = msgp.ReadBytesZC(metaBuf)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return buf
+	}
+	//   =  input - current pos
+	ends := len(buf) - len(metaBuf)
+	if ends > len(buf) {
+		return buf
+	}
+	return buf[:ends]
 }
 
 // AddLegacy adds a legacy version, is only called when no prior
@@ -219,12 +560,69 @@ func (z *xlMetaV2) AddLegacy(m *xlMetaV1Object) error {
 }
 
 // Load unmarshal and load the entire message pack.
+// Note that references to the incoming buffer may be kept as data.
 func (z *xlMetaV2) Load(buf []byte) error {
-	if err := checkXL2V1(buf); err != nil {
-		return err
+	buf, _, minor, err := checkXL2V1(buf)
+	if err != nil {
+		return errFileCorrupt
 	}
-	_, err := z.UnmarshalMsg(buf[8:])
-	return err
+	switch minor {
+	case 0:
+		_, err = z.UnmarshalMsg(buf)
+		if err != nil {
+			return errFileCorrupt
+		}
+		return nil
+	case 1:
+		v, buf, err := msgp.ReadBytesZC(buf)
+		if err != nil {
+			return errFileCorrupt
+		}
+		_, err = z.UnmarshalMsg(v)
+		if err != nil {
+			return errFileCorrupt
+		}
+		// Add remaining data.
+		z.data = nil
+		if len(buf) > 0 {
+			z.data = buf
+			if err := z.data.validate(); err != nil {
+				return errFileCorrupt
+			}
+		}
+	default:
+		return errors.New("unknown metadata version")
+	}
+	return nil
+}
+
+// AppendTo will marshal the data in z and append it to the provided slice.
+func (z *xlMetaV2) AppendTo(dst []byte) ([]byte, error) {
+	sz := len(xlHeader) + len(xlVersionCurrent) + msgp.ArrayHeaderSize + z.Msgsize() + len(z.data) + len(dst)
+	if cap(dst) < sz {
+		buf := make([]byte, len(dst), sz)
+		copy(buf, dst)
+		dst = buf
+	}
+	if err := z.data.validate(); err != nil {
+		return nil, err
+	}
+
+	dst = append(dst, xlHeader[:]...)
+	dst = append(dst, xlVersionCurrent[:]...)
+	// Add "bin 32" type header to always have enough space.
+	// We will fill out the correct size when we know it.
+	dst = append(dst, 0xc6, 0, 0, 0, 0)
+	dataOffset := len(dst)
+	dst, err := z.MarshalMsg(dst)
+	if err != nil {
+		return nil, err
+	}
+
+	// Update size...
+	binary.BigEndian.PutUint32(dst[dataOffset-4:dataOffset], uint32(len(dst)-dataOffset))
+
+	return append(dst, z.data...), nil
 }
 
 // AddVersion adds a new version
@@ -304,6 +702,10 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 				ventry.ObjectV2.MetaUser[k] = v
 			}
 		}
+		// If asked to save data.
+		if len(fi.Data) > 0 || fi.Size == 0 {
+			z.data.replace(dd.String(), fi.Data)
+		}
 	}
 
 	if !ventry.Valid() {
@@ -324,7 +726,7 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 				return nil
 			}
 		case ObjectType:
-			if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
+			if version.ObjectV2.VersionID == uv {
 				z.Versions[i] = ventry
 				return nil
 			}
@@ -332,7 +734,7 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 			// Allowing delete marker to replaced with an proper
 			// object data type as well, this is not S3 complaint
 			// behavior but kept here for future flexibility.
-			if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) {
+			if version.DeleteMarker.VersionID == uv {
 				z.Versions[i] = ventry
 				return nil
 			}
@@ -352,7 +754,7 @@ func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error)
 	versionID := ""
 	var uv uuid.UUID
 	// check if the version is not "null"
-	if !bytes.Equal(j.VersionID[:], uv[:]) {
+	if j.VersionID != uv {
 		versionID = uuid.UUID(j.VersionID).String()
 	}
 	fi := FileInfo{
@@ -516,7 +918,7 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 				return version.ObjectV1.DataDir, len(z.Versions) == 0, nil
 			}
 		case DeleteType:
-			if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) {
+			if version.DeleteMarker.VersionID == uv {
 				if updateVersion {
 					if len(z.Versions[i].DeleteMarker.MetaSys) == 0 {
 						z.Versions[i].DeleteMarker.MetaSys = make(map[string][]byte)
@@ -538,7 +940,7 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 				return "", len(z.Versions) == 0, nil
 			}
 		case ObjectType:
-			if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) && updateVersion {
+			if version.ObjectV2.VersionID == uv && updateVersion {
 				z.Versions[i].ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus)
 				return "", len(z.Versions) == 0, nil
 			}
@@ -550,7 +952,7 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		for _, version := range versions {
 			switch version.Type {
 			case ObjectType:
-				if bytes.Equal(version.ObjectV2.DataDir[:], dataDir[:]) {
+				if version.ObjectV2.DataDir == dataDir {
 					sameDataDirCount++
 				}
 			}
@@ -564,7 +966,7 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		}
 		switch version.Type {
 		case ObjectType:
-			if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
+			if version.ObjectV2.VersionID == uv {
 				if fi.TransitionStatus != "" {
 					z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+"transition-status"] = []byte(fi.TransitionStatus)
 					return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil