Store object EC in metadata header (#19534)

Keep the EC in header, so it can be retrieved easily for dynamic quorum calculations.

To not force a full metadata decode on every read the value will be 0/0 for data written in previous versions.

Size is expected to increase by 2 bytes per version, since all valid values can be represented with 1 byte each.

Example:
```
λ xl-meta xl.meta
{
  "Versions": [
    {
      "Header": {
        "EcM": 4,
        "EcN": 8,
        "Flags": 6,
        "ModTime": "2024-04-17T11:46:25.325613+02:00",
        "Signature": "0a409875",
        "Type": 1,
        "VersionID": "8e03504e11234957b2727bc53eda0d55"
      },
...
```

Not used for operations yet.

cmd/metacache-entries.go

@@ -120,6 +120,16 @@ func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *me
 	for i, eVer := range eVers.versions {
 		oVer := oVers.versions[i]
 		if eVer.header != oVer.header {
+			if eVer.header.hasEC() != oVer.header.hasEC() {
+				// One version has EC and the other doesn't - may have been written later.
+				// Compare without considering EC.
+				a, b := eVer.header, oVer.header
+				a.EcN, a.EcM = 0, 0
+				b.EcN, b.EcM = 0, 0
+				if a == b {
+					continue
+				}
+			}
 			if !strict && eVer.header.matchesNotStrict(oVer.header) {
 				if prefer == nil {
 					if eVer.header.sortsBefore(oVer.header) {

cmd/xl-storage-format-v2-legacy.go

@@ -29,6 +29,9 @@ func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err e
 	switch v {
 	case 1:
 		return x.unmarshalV1(bts)
+	case 2:
+		x2 := xlMetaV2VersionHeaderV2{xlMetaV2VersionHeader: x}
+		return x2.UnmarshalMsg(bts)
 	case xlHeaderVersion:
 		return x.UnmarshalMsg(bts)
 	}
@@ -123,3 +126,107 @@ func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error)
 	}
 	return o, err
 }
+
+// xlMetaV2VersionHeaderV2 is a version 2 of xlMetaV2VersionHeader before EcN and EcM were added.
+type xlMetaV2VersionHeaderV2 struct {
+	*xlMetaV2VersionHeader
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	z.EcN, z.EcN = 0, 0
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 5 {
+		err = msgp.ArrayError{Wanted: 5, Got: zb0001}
+		return
+	}
+	bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
+	if err != nil {
+		err = msgp.WrapError(err, "VersionID")
+		return
+	}
+	z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ModTime")
+		return
+	}
+	bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
+	if err != nil {
+		err = msgp.WrapError(err, "Signature")
+		return
+	}
+	{
+		var zb0002 uint8
+		zb0002, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Type")
+			return
+		}
+		z.Type = VersionType(zb0002)
+	}
+	{
+		var zb0003 uint8
+		zb0003, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "Flags")
+			return
+		}
+		z.Flags = xlFlags(zb0003)
+	}
+	o = bts
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) {
+	z.EcN, z.EcN = 0, 0
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 5 {
+		err = msgp.ArrayError{Wanted: 5, Got: zb0001}
+		return
+	}
+	err = dc.ReadExactBytes((z.VersionID)[:])
+	if err != nil {
+		err = msgp.WrapError(err, "VersionID")
+		return
+	}
+	z.ModTime, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "ModTime")
+		return
+	}
+	err = dc.ReadExactBytes((z.Signature)[:])
+	if err != nil {
+		err = msgp.WrapError(err, "Signature")
+		return
+	}
+	{
+		var zb0002 uint8
+		zb0002, err = dc.ReadUint8()
+		if err != nil {
+			err = msgp.WrapError(err, "Type")
+			return
+		}
+		z.Type = VersionType(zb0002)
+	}
+	{
+		var zb0003 uint8
+		zb0003, err = dc.ReadUint8()
+		if err != nil {
+			err = msgp.WrapError(err, "Flags")
+			return
+		}
+		z.Flags = xlFlags(zb0003)
+	}
+	return
+}

cmd/xl-storage-format-v2.go

@@ -250,15 +250,17 @@ type xlMetaV2VersionHeader struct {
 	Signature [4]byte
 	Type      VersionType
 	Flags     xlFlags
+	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 func (x xlMetaV2VersionHeader) String() string {
-	return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s",
+	return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s, N: %d, M: %d",
 		x.Type.String(),
 		hex.EncodeToString(x.VersionID[:]),
 		hex.EncodeToString(x.Signature[:]),
 		time.Unix(0, x.ModTime),
 		x.Flags.String(),
+		x.EcN, x.EcM,
 	)
 }
 
@@ -274,6 +276,11 @@ func (x xlMetaV2VersionHeader) matchesNotStrict(o xlMetaV2VersionHeader) bool {
 		x.Type == o.Type
 }
 
+// hasEC will return true if the version has erasure coding information.
+func (x xlMetaV2VersionHeader) hasEC() bool {
+	return x.EcM > 0 && x.EcN > 0
+}
+
 // sortsBefore can be used as a tiebreaker for stable sorting/selecting.
 // Returns false on ties.
 func (x xlMetaV2VersionHeader) sortsBefore(o xlMetaV2VersionHeader) bool {
@@ -351,12 +358,18 @@ func (j *xlMetaV2Version) header() xlMetaV2VersionHeader {
 	if j.Type == ObjectType && j.ObjectV2.InlineData() {
 		flags |= xlFlagInlineData
 	}
+	var ecM, ecN uint8
+	if j.Type == ObjectType && j.ObjectV2 != nil {
+		ecM, ecN = uint8(j.ObjectV2.ErasureM), uint8(j.ObjectV2.ErasureN)
+	}
 	return xlMetaV2VersionHeader{
 		VersionID: j.getVersionID(),
 		ModTime:   j.getModTime().UnixNano(),
 		Signature: j.getSignature(),
 		Type:      j.Type,
 		Flags:     flags,
+		EcN:       ecM,
+		EcM:       ecN,
 	}
 }
 
@@ -440,7 +453,7 @@ func (j *xlMetaV2Version) ToFileInfo(volume, path string, allParts bool) (fi Fil
 }
 
 const (
-	xlHeaderVersion = 2
+	xlHeaderVersion = 3
 	xlMetaVersion   = 2
 )
 

cmd/xl-storage-format-v2_gen.go

@@ -2198,8 +2198,8 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 5 {
+	if zb0001 != 7 {
-		err = msgp.ArrayError{Wanted: 5, Got: zb0001}
+		err = msgp.ArrayError{Wanted: 7, Got: zb0001}
 		return
 	}
 	err = dc.ReadExactBytes((z.VersionID)[:])
@@ -2235,13 +2235,23 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
+	z.EcM, err = dc.ReadUint8()
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
+	z.EcN, err = dc.ReadUint8()
+	if err != nil {
+		err = msgp.WrapError(err, "EcN")
+		return
+	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
 func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 5
+	// array header, size 7
-	err = en.Append(0x95)
+	err = en.Append(0x97)
 	if err != nil {
 		return
 	}
@@ -2270,19 +2280,31 @@ func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Flags")
 		return
 	}
+	err = en.WriteUint8(z.EcM)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
+	err = en.WriteUint8(z.EcN)
+	if err != nil {
+		err = msgp.WrapError(err, "EcN")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 5
+	// array header, size 7
-	o = append(o, 0x95)
+	o = append(o, 0x97)
 	o = msgp.AppendBytes(o, (z.VersionID)[:])
 	o = msgp.AppendInt64(o, z.ModTime)
 	o = msgp.AppendBytes(o, (z.Signature)[:])
 	o = msgp.AppendUint8(o, uint8(z.Type))
 	o = msgp.AppendUint8(o, uint8(z.Flags))
+	o = msgp.AppendUint8(o, z.EcM)
+	o = msgp.AppendUint8(o, z.EcN)
 	return
 }
 
@@ -2294,8 +2316,8 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 5 {
+	if zb0001 != 7 {
-		err = msgp.ArrayError{Wanted: 5, Got: zb0001}
+		err = msgp.ArrayError{Wanted: 7, Got: zb0001}
 		return
 	}
 	bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
@@ -2331,12 +2353,22 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
+	z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
+	z.EcN, bts, err = msgp.ReadUint8Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "EcN")
+		return
+	}
 	o = bts
 	return
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *xlMetaV2VersionHeader) Msgsize() (s int) {
-	s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size
+	s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size
 	return
 }

docs/debugging/xl-meta/main.go

@@ -138,7 +138,7 @@ FLAGS:
 					b = nbuf
 				}
 
-				nVers, v, err := decodeXLHeaders(v)
+				hdr, v, err := decodeXLHeaders(v)
 				if err != nil {
 					return nil, err
 				}
@@ -147,10 +147,11 @@ FLAGS:
 					Header   json.RawMessage
 					Metadata json.RawMessage
 				}
-				versions := make([]version, nVers)
+				versions := make([]version, hdr.versions)
-				err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
+				headerVer := hdr.headerVer
+				err = decodeVersions(v, hdr.versions, func(idx int, hdr, meta []byte) error {
 					var header xlMetaV2VersionHeaderV2
-					if _, err := header.UnmarshalMsg(hdr); err != nil {
+					if _, err := header.UnmarshalMsg(hdr, headerVer); err != nil {
 						return err
 					}
 					b, err := header.MarshalJSON()
@@ -533,33 +534,38 @@ func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
 }
 
 const (
-	xlHeaderVersion = 2
+	xlHeaderVersion = 3
 	xlMetaVersion   = 2
 )
 
-func decodeXLHeaders(buf []byte) (versions int, b []byte, err error) {
+type xlHeaders struct {
-	hdrVer, buf, err := msgp.ReadUintBytes(buf)
+	versions           int
+	headerVer, metaVer uint
+}
+
+func decodeXLHeaders(buf []byte) (x xlHeaders, b []byte, err error) {
+	x.headerVer, buf, err = msgp.ReadUintBytes(buf)
 	if err != nil {
-		return 0, buf, err
+		return x, buf, err
 	}
-	metaVer, buf, err := msgp.ReadUintBytes(buf)
+	x.metaVer, buf, err = msgp.ReadUintBytes(buf)
 	if err != nil {
-		return 0, buf, err
+		return x, buf, err
 	}
-	if hdrVer > xlHeaderVersion {
+	if x.headerVer > xlHeaderVersion {
-		return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer)
+		return x, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", x.headerVer)
 	}
-	if metaVer > xlMetaVersion {
+	if x.metaVer > xlMetaVersion {
-		return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer)
+		return x, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", x.metaVer)
 	}
-	versions, buf, err = msgp.ReadIntBytes(buf)
+	x.versions, buf, err = msgp.ReadIntBytes(buf)
 	if err != nil {
-		return 0, buf, err
+		return x, buf, err
 	}
-	if versions < 0 {
+	if x.versions < 0 {
-		return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions)
+		return x, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", x.versions)
 	}
-	return versions, buf, nil
+	return x, buf, nil
 }
 
 // decodeVersions will decode a number of versions from a buffer
@@ -589,18 +595,23 @@ type xlMetaV2VersionHeaderV2 struct {
 	Signature [4]byte
 	Type      uint8
 	Flags     uint8
+	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byte, err error) {
 	var zb0001 uint32
 	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 5 {
+	want := uint32(5)
-		err = msgp.ArrayError{Wanted: 5, Got: zb0001}
+	if hdrVer > 2 {
+		want += 2
+	}
+	if zb0001 != want {
+		err = msgp.ArrayError{Wanted: want, Got: zb0001}
 		return
 	}
 	bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
@@ -636,6 +647,27 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error)
 		}
 		z.Flags = zb0003
 	}
+	if hdrVer > 2 {
+		// Version 3 has EcM and EcN
+		{
+			var zb0004 uint8
+			zb0004, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "EcM")
+				return
+			}
+			z.EcM = zb0004
+		}
+		{
+			var zb0005 uint8
+			zb0005, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "EcN")
+				return
+			}
+			z.EcN = zb0005
+		}
+	}
 	o = bts
 	return
 }
@@ -647,12 +679,15 @@ func (z xlMetaV2VersionHeaderV2) MarshalJSON() (o []byte, err error) {
 		Signature string
 		Type      uint8
 		Flags     uint8
+		EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 	}{
 		VersionID: hex.EncodeToString(z.VersionID[:]),
 		ModTime:   time.Unix(0, z.ModTime),
 		Signature: hex.EncodeToString(z.Signature[:]),
 		Type:      z.Type,
 		Flags:     z.Flags,
+		EcM:       z.EcM,
+		EcN:       z.EcN,
 	}
 	return json.Marshal(tmp)
 }