Inline small file data in xl.meta file (#11758)

docs/bucket/versioning/xl-meta.go

@@ -18,9 +18,12 @@ package main
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 
@@ -28,8 +31,6 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-var xlHeader = [4]byte{'X', 'L', '2', ' '}
-
 func main() {
 	app := cli.NewApp()
 	app.Copyright = "MinIO, Inc."
@@ -53,6 +54,10 @@ GLOBAL FLAGS:
 			Usage: "Print each file as a separate line without formatting",
 			Name:  "ndjson",
 		},
+		cli.BoolFlag{
+			Usage: "Display inline data keys and sizes",
+			Name:  "data",
+		},
 	}
 
 	app.Action = func(c *cli.Context) error {
@@ -75,39 +80,57 @@ GLOBAL FLAGS:
 				r = f
 			}
 
-			// Read header
-			var tmp [4]byte
-			_, err := io.ReadFull(r, tmp[:])
+			b, err := ioutil.ReadAll(r)
 			if err != nil {
 				return err
 			}
-			if !bytes.Equal(tmp[:], xlHeader[:]) {
-				return fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], tmp[:4])
-			}
-			// Skip version check for now
-			_, err = io.ReadFull(r, tmp[:])
+			b, _, minor, err := checkXL2V1(b)
 			if err != nil {
 				return err
 			}
 
-			var buf bytes.Buffer
-			_, err = msgp.CopyToJSON(&buf, r)
-			if err != nil {
-				return err
+			buf := bytes.NewBuffer(nil)
+			var data xlMetaInlineData
+			switch minor {
+			case 0:
+				_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b))
+				if err != nil {
+					return err
+				}
+			case 1:
+				v, b, err := msgp.ReadBytesZC(b)
+				if err != nil {
+					return err
+				}
+				_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v))
+				if err != nil {
+					return err
+				}
+				data = b
+			default:
+				return errors.New("unknown metadata version")
+			}
+
+			if c.Bool("data") {
+				b, err := data.json()
+				if err != nil {
+					return err
+				}
+				buf = bytes.NewBuffer(b)
 			}
 			if c.Bool("ndjson") {
 				fmt.Println(buf.String())
 				continue
 			}
 			var msi map[string]interface{}
-			dec := json.NewDecoder(&buf)
+			dec := json.NewDecoder(buf)
 			// Use number to preserve integers.
 			dec.UseNumber()
 			err = dec.Decode(&msi)
 			if err != nil {
 				return err
 			}
-			b, err := json.MarshalIndent(msi, "", "  ")
+			b, err = json.MarshalIndent(msi, "", "  ")
 			if err != nil {
 				return err
 			}
@@ -120,3 +143,111 @@ GLOBAL FLAGS:
 		log.Fatal(err)
 	}
 }
+
+var (
+	// XL header specifies the format
+	xlHeader = [4]byte{'X', 'L', '2', ' '}
+
+	// Current version being written.
+	xlVersionCurrent [4]byte
+)
+
+const (
+	// Breaking changes.
+	// Newer versions cannot be read by older software.
+	// This will prevent downgrades to incompatible versions.
+	xlVersionMajor = 1
+
+	// Non breaking changes.
+	// Bumping this is informational, but should be done
+	// if any change is made to the data stored, bumping this
+	// will allow to detect the exact version later.
+	xlVersionMinor = 1
+)
+
+func init() {
+	binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor)
+	binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor)
+}
+
+// checkXL2V1 will check if the metadata has correct header and is a known major version.
+// The remaining payload and versions are returned.
+func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) {
+	if len(buf) <= 8 {
+		return payload, 0, 0, fmt.Errorf("xlMeta: no data")
+	}
+
+	if !bytes.Equal(buf[:4], xlHeader[:]) {
+		return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4])
+	}
+
+	if bytes.Equal(buf[4:8], []byte("1   ")) {
+		// Set as 1,0.
+		major, minor = 1, 0
+	} else {
+		major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8])
+	}
+	if major > xlVersionMajor {
+		return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major)
+	}
+
+	return buf[8:], major, minor, nil
+}
+
+const xlMetaInlineDataVer = 1
+
+type xlMetaInlineData []byte
+
+// afterVersion returns the payload after the version, if any.
+func (x xlMetaInlineData) afterVersion() []byte {
+	if len(x) == 0 {
+		return x
+	}
+	return x[1:]
+}
+
+// versionOK returns whether the version is ok.
+func (x xlMetaInlineData) versionOK() bool {
+	if len(x) == 0 {
+		return true
+	}
+	return x[0] > 0 && x[0] <= xlMetaInlineDataVer
+}
+
+func (x xlMetaInlineData) json() ([]byte, error) {
+	if len(x) == 0 {
+		return []byte("{}"), nil
+	}
+	if !x.versionOK() {
+		return nil, errors.New("xlMetaInlineData: unknown version")
+	}
+
+	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
+	if err != nil {
+		return nil, err
+	}
+	res := []byte("{")
+
+	for i := uint32(0); i < sz; i++ {
+		var key, val []byte
+		key, buf, err = msgp.ReadMapKeyZC(buf)
+		if err != nil {
+			return nil, err
+		}
+		if len(key) == 0 {
+			return nil, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
+		}
+		// Skip data...
+		val, buf, err = msgp.ReadBytesZC(buf)
+		if err != nil {
+			return nil, err
+		}
+		if i > 0 {
+			res = append(res, ',')
+		}
+		s := fmt.Sprintf(`"%s":%d`, string(key), len(val))
+		res = append(res, []byte(s)...)
+	}
+	res = append(res, '}')
+	return res, nil
+}