re-arrange metacache struct to be optimal (#13609)

cmd/data-scanner.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"io/fs"
 	"math"
 	"math/rand"
 	"net/http"
@@ -816,14 +817,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 // scannerItem represents each file while walking.
 type scannerItem struct {
-	Path string
+	Path        string
-	Typ  os.FileMode
-
 	bucket      string // Bucket.
 	prefix      string // Only the prefix if any, does not have final object name.
 	objectName  string // Only the object name without prefixes.
-	lifeCycle   *lifecycle.Lifecycle
 	replication replicationConfig
+	lifeCycle   *lifecycle.Lifecycle
+	Typ         fs.FileMode
 	heal        bool // Has the object been selected for heal check?
 	debug       bool
 }

cmd/metacache-stream.go

@@ -56,14 +56,13 @@ const metacacheStreamVersion = 2
 
 // metacacheWriter provides a serializer of metacache objects.
 type metacacheWriter struct {
+	streamErr   error
 	mw          *msgp.Writer
 	creator     func() error
 	closer      func() error
 	blockSize   int
+	streamWg    sync.WaitGroup
 	reuseBlocks bool
-
-	streamErr error
-	streamWg  sync.WaitGroup
 }
 
 // newMetacacheWriter will create a serializer that will write objects in given order to the output.

cmd/metacache.go

@@ -56,18 +56,21 @@ const (
 
 // metacache contains a tracked cache entry.
 type metacache struct {
-	id           string     `msg:"id"`
+	// do not re-arrange the struct this struct has been ordered to use less
-	bucket       string     `msg:"b"`
+	// space - if you do so please run https://github.com/orijtech/structslop
-	root         string     `msg:"root"`
+	// and verify if your changes are optimal.
-	recursive    bool       `msg:"rec"`
-	filter       string     `msg:"flt"`
-	status       scanStatus `msg:"stat"`
-	fileNotFound bool       `msg:"fnf"`
-	error        string     `msg:"err"`
-	started      time.Time  `msg:"st"`
 	ended        time.Time  `msg:"end"`
-	lastUpdate   time.Time  `msg:"u"`
+	started      time.Time  `msg:"st"`
 	lastHandout  time.Time  `msg:"lh"`
+	lastUpdate   time.Time  `msg:"u"`
+	bucket       string     `msg:"b"`
+	filter       string     `msg:"flt"`
+	id           string     `msg:"id"`
+	error        string     `msg:"err"`
+	root         string     `msg:"root"`
+	fileNotFound bool       `msg:"fnf"`
+	status       scanStatus `msg:"stat"`
+	recursive    bool       `msg:"rec"`
 	dataVersion  uint8      `msg:"v"`
 }
 

cmd/metacache_gen.go

@@ -24,10 +24,28 @@ func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "id":
+		case "end":
-			z.id, err = dc.ReadString()
+			z.ended, err = dc.ReadTime()
 			if err != nil {
-				err = msgp.WrapError(err, "id")
+				err = msgp.WrapError(err, "ended")
+				return
+			}
+		case "st":
+			z.started, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "lh":
+			z.lastHandout, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "lastHandout")
+				return
+			}
+		case "u":
+			z.lastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "lastUpdate")
 				return
 			}
 		case "b":
@@ -36,22 +54,34 @@ func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "bucket")
 				return
 			}
+		case "flt":
+			z.filter, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "filter")
+				return
+			}
+		case "id":
+			z.id, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "id")
+				return
+			}
+		case "err":
+			z.error, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "error")
+				return
+			}
 		case "root":
 			z.root, err = dc.ReadString()
 			if err != nil {
 				err = msgp.WrapError(err, "root")
 				return
 			}
-		case "rec":
+		case "fnf":
-			z.recursive, err = dc.ReadBool()
+			z.fileNotFound, err = dc.ReadBool()
 			if err != nil {
-				err = msgp.WrapError(err, "recursive")
+				err = msgp.WrapError(err, "fileNotFound")
-				return
-			}
-		case "flt":
-			z.filter, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "filter")
 				return
 			}
 		case "stat":
@@ -64,40 +94,10 @@ func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.status = scanStatus(zb0002)
 			}
-		case "fnf":
+		case "rec":
-			z.fileNotFound, err = dc.ReadBool()
+			z.recursive, err = dc.ReadBool()
 			if err != nil {
-				err = msgp.WrapError(err, "fileNotFound")
+				err = msgp.WrapError(err, "recursive")
-				return
-			}
-		case "err":
-			z.error, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "error")
-				return
-			}
-		case "st":
-			z.started, err = dc.ReadTime()
-			if err != nil {
-				err = msgp.WrapError(err, "started")
-				return
-			}
-		case "end":
-			z.ended, err = dc.ReadTime()
-			if err != nil {
-				err = msgp.WrapError(err, "ended")
-				return
-			}
-		case "u":
-			z.lastUpdate, err = dc.ReadTime()
-			if err != nil {
-				err = msgp.WrapError(err, "lastUpdate")
-				return
-			}
-		case "lh":
-			z.lastHandout, err = dc.ReadTime()
-			if err != nil {
-				err = msgp.WrapError(err, "lastHandout")
 				return
 			}
 		case "v":
@@ -120,84 +120,14 @@ func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 13
-	// write "id"
+	// write "end"
-	err = en.Append(0x8d, 0xa2, 0x69, 0x64)
+	err = en.Append(0x8d, 0xa3, 0x65, 0x6e, 0x64)
 	if err != nil {
 		return
 	}
-	err = en.WriteString(z.id)
+	err = en.WriteTime(z.ended)
 	if err != nil {
-		err = msgp.WrapError(err, "id")
+		err = msgp.WrapError(err, "ended")
-		return
-	}
-	// write "b"
-	err = en.Append(0xa1, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.bucket)
-	if err != nil {
-		err = msgp.WrapError(err, "bucket")
-		return
-	}
-	// write "root"
-	err = en.Append(0xa4, 0x72, 0x6f, 0x6f, 0x74)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.root)
-	if err != nil {
-		err = msgp.WrapError(err, "root")
-		return
-	}
-	// write "rec"
-	err = en.Append(0xa3, 0x72, 0x65, 0x63)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.recursive)
-	if err != nil {
-		err = msgp.WrapError(err, "recursive")
-		return
-	}
-	// write "flt"
-	err = en.Append(0xa3, 0x66, 0x6c, 0x74)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.filter)
-	if err != nil {
-		err = msgp.WrapError(err, "filter")
-		return
-	}
-	// write "stat"
-	err = en.Append(0xa4, 0x73, 0x74, 0x61, 0x74)
-	if err != nil {
-		return
-	}
-	err = en.WriteUint8(uint8(z.status))
-	if err != nil {
-		err = msgp.WrapError(err, "status")
-		return
-	}
-	// write "fnf"
-	err = en.Append(0xa3, 0x66, 0x6e, 0x66)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.fileNotFound)
-	if err != nil {
-		err = msgp.WrapError(err, "fileNotFound")
-		return
-	}
-	// write "err"
-	err = en.Append(0xa3, 0x65, 0x72, 0x72)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.error)
-	if err != nil {
-		err = msgp.WrapError(err, "error")
 		return
 	}
 	// write "st"
@@ -210,14 +140,14 @@ func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "started")
 		return
 	}
-	// write "end"
+	// write "lh"
-	err = en.Append(0xa3, 0x65, 0x6e, 0x64)
+	err = en.Append(0xa2, 0x6c, 0x68)
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.ended)
+	err = en.WriteTime(z.lastHandout)
 	if err != nil {
-		err = msgp.WrapError(err, "ended")
+		err = msgp.WrapError(err, "lastHandout")
 		return
 	}
 	// write "u"
@@ -230,14 +160,84 @@ func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "lastUpdate")
 		return
 	}
-	// write "lh"
+	// write "b"
-	err = en.Append(0xa2, 0x6c, 0x68)
+	err = en.Append(0xa1, 0x62)
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.lastHandout)
+	err = en.WriteString(z.bucket)
 	if err != nil {
-		err = msgp.WrapError(err, "lastHandout")
+		err = msgp.WrapError(err, "bucket")
+		return
+	}
+	// write "flt"
+	err = en.Append(0xa3, 0x66, 0x6c, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.filter)
+	if err != nil {
+		err = msgp.WrapError(err, "filter")
+		return
+	}
+	// write "id"
+	err = en.Append(0xa2, 0x69, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.id)
+	if err != nil {
+		err = msgp.WrapError(err, "id")
+		return
+	}
+	// write "err"
+	err = en.Append(0xa3, 0x65, 0x72, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.error)
+	if err != nil {
+		err = msgp.WrapError(err, "error")
+		return
+	}
+	// write "root"
+	err = en.Append(0xa4, 0x72, 0x6f, 0x6f, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.root)
+	if err != nil {
+		err = msgp.WrapError(err, "root")
+		return
+	}
+	// write "fnf"
+	err = en.Append(0xa3, 0x66, 0x6e, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.fileNotFound)
+	if err != nil {
+		err = msgp.WrapError(err, "fileNotFound")
+		return
+	}
+	// write "stat"
+	err = en.Append(0xa4, 0x73, 0x74, 0x61, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(uint8(z.status))
+	if err != nil {
+		err = msgp.WrapError(err, "status")
+		return
+	}
+	// write "rec"
+	err = en.Append(0xa3, 0x72, 0x65, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.recursive)
+	if err != nil {
+		err = msgp.WrapError(err, "recursive")
 		return
 	}
 	// write "v"
@@ -257,42 +257,42 @@ func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *metacache) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 13
-	// string "id"
+	// string "end"
-	o = append(o, 0x8d, 0xa2, 0x69, 0x64)
+	o = append(o, 0x8d, 0xa3, 0x65, 0x6e, 0x64)
-	o = msgp.AppendString(o, z.id)
+	o = msgp.AppendTime(o, z.ended)
-	// string "b"
-	o = append(o, 0xa1, 0x62)
-	o = msgp.AppendString(o, z.bucket)
-	// string "root"
-	o = append(o, 0xa4, 0x72, 0x6f, 0x6f, 0x74)
-	o = msgp.AppendString(o, z.root)
-	// string "rec"
-	o = append(o, 0xa3, 0x72, 0x65, 0x63)
-	o = msgp.AppendBool(o, z.recursive)
-	// string "flt"
-	o = append(o, 0xa3, 0x66, 0x6c, 0x74)
-	o = msgp.AppendString(o, z.filter)
-	// string "stat"
-	o = append(o, 0xa4, 0x73, 0x74, 0x61, 0x74)
-	o = msgp.AppendUint8(o, uint8(z.status))
-	// string "fnf"
-	o = append(o, 0xa3, 0x66, 0x6e, 0x66)
-	o = msgp.AppendBool(o, z.fileNotFound)
-	// string "err"
-	o = append(o, 0xa3, 0x65, 0x72, 0x72)
-	o = msgp.AppendString(o, z.error)
 	// string "st"
 	o = append(o, 0xa2, 0x73, 0x74)
 	o = msgp.AppendTime(o, z.started)
-	// string "end"
-	o = append(o, 0xa3, 0x65, 0x6e, 0x64)
-	o = msgp.AppendTime(o, z.ended)
-	// string "u"
-	o = append(o, 0xa1, 0x75)
-	o = msgp.AppendTime(o, z.lastUpdate)
 	// string "lh"
 	o = append(o, 0xa2, 0x6c, 0x68)
 	o = msgp.AppendTime(o, z.lastHandout)
+	// string "u"
+	o = append(o, 0xa1, 0x75)
+	o = msgp.AppendTime(o, z.lastUpdate)
+	// string "b"
+	o = append(o, 0xa1, 0x62)
+	o = msgp.AppendString(o, z.bucket)
+	// string "flt"
+	o = append(o, 0xa3, 0x66, 0x6c, 0x74)
+	o = msgp.AppendString(o, z.filter)
+	// string "id"
+	o = append(o, 0xa2, 0x69, 0x64)
+	o = msgp.AppendString(o, z.id)
+	// string "err"
+	o = append(o, 0xa3, 0x65, 0x72, 0x72)
+	o = msgp.AppendString(o, z.error)
+	// string "root"
+	o = append(o, 0xa4, 0x72, 0x6f, 0x6f, 0x74)
+	o = msgp.AppendString(o, z.root)
+	// string "fnf"
+	o = append(o, 0xa3, 0x66, 0x6e, 0x66)
+	o = msgp.AppendBool(o, z.fileNotFound)
+	// string "stat"
+	o = append(o, 0xa4, 0x73, 0x74, 0x61, 0x74)
+	o = msgp.AppendUint8(o, uint8(z.status))
+	// string "rec"
+	o = append(o, 0xa3, 0x72, 0x65, 0x63)
+	o = msgp.AppendBool(o, z.recursive)
 	// string "v"
 	o = append(o, 0xa1, 0x76)
 	o = msgp.AppendUint8(o, z.dataVersion)
@@ -317,10 +317,28 @@ func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "id":
+		case "end":
-			z.id, bts, err = msgp.ReadStringBytes(bts)
+			z.ended, bts, err = msgp.ReadTimeBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "id")
+				err = msgp.WrapError(err, "ended")
+				return
+			}
+		case "st":
+			z.started, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "lh":
+			z.lastHandout, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "lastHandout")
+				return
+			}
+		case "u":
+			z.lastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "lastUpdate")
 				return
 			}
 		case "b":
@@ -329,22 +347,34 @@ func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "bucket")
 				return
 			}
+		case "flt":
+			z.filter, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "filter")
+				return
+			}
+		case "id":
+			z.id, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "id")
+				return
+			}
+		case "err":
+			z.error, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "error")
+				return
+			}
 		case "root":
 			z.root, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "root")
 				return
 			}
-		case "rec":
+		case "fnf":
-			z.recursive, bts, err = msgp.ReadBoolBytes(bts)
+			z.fileNotFound, bts, err = msgp.ReadBoolBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "recursive")
+				err = msgp.WrapError(err, "fileNotFound")
-				return
-			}
-		case "flt":
-			z.filter, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "filter")
 				return
 			}
 		case "stat":
@@ -357,40 +387,10 @@ func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 				z.status = scanStatus(zb0002)
 			}
-		case "fnf":
+		case "rec":
-			z.fileNotFound, bts, err = msgp.ReadBoolBytes(bts)
+			z.recursive, bts, err = msgp.ReadBoolBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "fileNotFound")
+				err = msgp.WrapError(err, "recursive")
-				return
-			}
-		case "err":
-			z.error, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "error")
-				return
-			}
-		case "st":
-			z.started, bts, err = msgp.ReadTimeBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "started")
-				return
-			}
-		case "end":
-			z.ended, bts, err = msgp.ReadTimeBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "ended")
-				return
-			}
-		case "u":
-			z.lastUpdate, bts, err = msgp.ReadTimeBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "lastUpdate")
-				return
-			}
-		case "lh":
-			z.lastHandout, bts, err = msgp.ReadTimeBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "lastHandout")
 				return
 			}
 		case "v":
@@ -413,7 +413,7 @@ func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *metacache) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.id) + 2 + msgp.StringPrefixSize + len(z.bucket) + 5 + msgp.StringPrefixSize + len(z.root) + 4 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.filter) + 5 + msgp.Uint8Size + 4 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.error) + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 2 + msgp.TimeSize + 3 + msgp.TimeSize + 2 + msgp.Uint8Size
+	s = 1 + 4 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 2 + msgp.TimeSize + 2 + msgp.StringPrefixSize + len(z.bucket) + 4 + msgp.StringPrefixSize + len(z.filter) + 3 + msgp.StringPrefixSize + len(z.id) + 4 + msgp.StringPrefixSize + len(z.error) + 5 + msgp.StringPrefixSize + len(z.root) + 4 + msgp.BoolSize + 5 + msgp.Uint8Size + 4 + msgp.BoolSize + 2 + msgp.Uint8Size
 	return
 }