/* * MinIO Cloud Storage, (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "bytes" "encoding/binary" "encoding/json" "errors" "fmt" "io" "io/ioutil" "log" "os" "github.com/minio/cli" "github.com/tinylib/msgp/msgp" ) func main() { app := cli.NewApp() app.Copyright = "MinIO, Inc." app.Usage = "xl.meta to JSON" app.HideVersion = true app.CustomAppHelpTemplate = `NAME: {{.Name}} - {{.Usage}} USAGE: {{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES... {{if .VisibleFlags}} GLOBAL FLAGS: {{range .VisibleFlags}}{{.}} {{end}}{{end}} ` app.HideHelpCommand = true app.Flags = []cli.Flag{ cli.BoolFlag{ Usage: "Print each file as a separate line without formatting", Name: "ndjson", }, cli.BoolFlag{ Usage: "Display inline data keys and sizes", Name: "data", }, } app.Action = func(c *cli.Context) error { files := c.Args() if len(files) == 0 { // If no args, assume xl.meta files = []string{"xl.meta"} } for _, file := range files { var r io.Reader switch file { case "-": r = os.Stdin default: f, err := os.Open(file) if err != nil { return err } defer f.Close() r = f } b, err := ioutil.ReadAll(r) if err != nil { return err } b, _, minor, err := checkXL2V1(b) if err != nil { return err } buf := bytes.NewBuffer(nil) var data xlMetaInlineData switch minor { case 0: _, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b)) if err != nil { return err } case 1, 2: v, b, err := msgp.ReadBytesZC(b) if err != nil { return err } if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil { // Read metadata CRC (added in v2, ignore if not found) b = nbuf } _, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v)) if err != nil { return err } data = b default: return errors.New("unknown metadata version") } if c.Bool("data") { b, err := data.json() if err != nil { return err } buf = bytes.NewBuffer(b) } if c.Bool("ndjson") { fmt.Println(buf.String()) continue } var msi map[string]interface{} dec := json.NewDecoder(buf) // Use number to preserve integers. dec.UseNumber() err = dec.Decode(&msi) if err != nil { return err } b, err = json.MarshalIndent(msi, "", " ") if err != nil { return err } fmt.Println(string(b)) } return nil } err := app.Run(os.Args) if err != nil { log.Fatal(err) } } var ( // XL header specifies the format xlHeader = [4]byte{'X', 'L', '2', ' '} // Current version being written. xlVersionCurrent [4]byte ) const ( // Breaking changes. // Newer versions cannot be read by older software. // This will prevent downgrades to incompatible versions. xlVersionMajor = 1 // Non breaking changes. // Bumping this is informational, but should be done // if any change is made to the data stored, bumping this // will allow to detect the exact version later. xlVersionMinor = 1 ) func init() { binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor) binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor) } // checkXL2V1 will check if the metadata has correct header and is a known major version. // The remaining payload and versions are returned. func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) { if len(buf) <= 8 { return payload, 0, 0, fmt.Errorf("xlMeta: no data") } if !bytes.Equal(buf[:4], xlHeader[:]) { return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) } if bytes.Equal(buf[4:8], []byte("1 ")) { // Set as 1,0. major, minor = 1, 0 } else { major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) } if major > xlVersionMajor { return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) } return buf[8:], major, minor, nil } const xlMetaInlineDataVer = 1 type xlMetaInlineData []byte // afterVersion returns the payload after the version, if any. func (x xlMetaInlineData) afterVersion() []byte { if len(x) == 0 { return x } return x[1:] } // versionOK returns whether the version is ok. func (x xlMetaInlineData) versionOK() bool { if len(x) == 0 { return true } return x[0] > 0 && x[0] <= xlMetaInlineDataVer } func (x xlMetaInlineData) json() ([]byte, error) { if len(x) == 0 { return []byte("{}"), nil } if !x.versionOK() { return nil, errors.New("xlMetaInlineData: unknown version") } sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) if err != nil { return nil, err } res := []byte("{") for i := uint32(0); i < sz; i++ { var key, val []byte key, buf, err = msgp.ReadMapKeyZC(buf) if err != nil { return nil, err } if len(key) == 0 { return nil, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) } // Skip data... val, buf, err = msgp.ReadBytesZC(buf) if err != nil { return nil, err } if i > 0 { res = append(res, ',') } s := fmt.Sprintf(`"%s":%d`, string(key), len(val)) res = append(res, []byte(s)...) } res = append(res, '}') return res, nil }