// Copyright (c) 2015-2022 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package main import ( "bytes" "encoding/binary" "encoding/hex" "errors" "fmt" "io" "io/ioutil" "os" "strings" "time" json "github.com/minio/colorjson" "github.com/klauspost/compress/zip" "github.com/tinylib/msgp/msgp" ) func inspectToExportType(downloadPath string, datajson bool) error { decode := func(r io.Reader, file string) ([]byte, error) { b, e := ioutil.ReadAll(r) if e != nil { return nil, e } b, _, minor, e := checkXL2V1(b) if e != nil { return nil, e } buf := bytes.NewBuffer(nil) var data xlMetaInlineData switch minor { case 0: _, e = msgp.CopyToJSON(buf, bytes.NewReader(b)) if e != nil { return nil, e } case 1, 2: v, b, e := msgp.ReadBytesZC(b) if e != nil { return nil, e } if _, nbuf, e := msgp.ReadUint32Bytes(b); e == nil { // Read metadata CRC (added in v2, ignore if not found) b = nbuf } _, e = msgp.CopyToJSON(buf, bytes.NewReader(v)) if e != nil { return nil, e } data = b case 3: v, b, e := msgp.ReadBytesZC(b) if e != nil { return nil, e } if _, nbuf, e := msgp.ReadUint32Bytes(b); e == nil { // Read metadata CRC (added in v2, ignore if not found) b = nbuf } nVers, v, e := decodeXLHeaders(v) if e != nil { return nil, e } type version struct { Idx int Header json.RawMessage Metadata json.RawMessage } versions := make([]version, nVers) e = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error { var header xlMetaV2VersionHeaderV2 if _, e := header.UnmarshalMsg(hdr); e != nil { return e } b, e := header.MarshalJSON() if e != nil { return e } var buf bytes.Buffer if _, e := msgp.UnmarshalAsJSON(&buf, meta); e != nil { return e } versions[idx] = version{ Idx: idx, Header: b, Metadata: buf.Bytes(), } return nil }) if e != nil { return nil, e } enc := json.NewEncoder(buf) if e := enc.Encode(struct { Versions []version }{Versions: versions}); e != nil { return nil, e } data = b default: return nil, fmt.Errorf("unknown metadata version %d", minor) } if datajson { b, e := data.json() if e != nil { return nil, e } buf = bytes.NewBuffer(b) } return buf.Bytes(), nil } fmt.Println("{") hasWritten := false var r io.Reader var sz int64 f, e := os.Open(downloadPath) if e != nil { return e } if st, e := f.Stat(); e == nil { sz = st.Size() } defer f.Close() r = f zr, e := zip.NewReader(r.(io.ReaderAt), sz) if e != nil { return e } for _, file := range zr.File { if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") { r, e := file.Open() if e != nil { return e } // Quote string... b, _ := json.Marshal(file.Name) if hasWritten { fmt.Print(",\n") } fmt.Printf("\t%s: ", string(b)) b, e = decode(r, file.Name) if e != nil { return e } fmt.Print(string(b)) hasWritten = true } } fmt.Println("") fmt.Println("}") return nil } var ( // XL header specifies the format xlHeader = [4]byte{'X', 'L', '2', ' '} // Current version being written. xlVersionCurrent [4]byte ) const ( // Breaking changes. // Newer versions cannot be read by older software. // This will prevent downgrades to incompatible versions. xlVersionMajor = 1 // Non breaking changes. // Bumping this is informational, but should be done // if any change is made to the data stored, bumping this // will allow to detect the exact version later. xlVersionMinor = 1 ) func init() { binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor) binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor) } // checkXL2V1 will check if the metadata has correct header and is a known major version. // The remaining payload and versions are returned. func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, e error) { if len(buf) <= 8 { return payload, 0, 0, fmt.Errorf("xlMeta: no data") } if !bytes.Equal(buf[:4], xlHeader[:]) { return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) } if bytes.Equal(buf[4:8], []byte("1 ")) { // Set as 1,0. major, minor = 1, 0 } else { major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) } if major > xlVersionMajor { return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) } return buf[8:], major, minor, nil } const xlMetaInlineDataVer = 1 type xlMetaInlineData []byte // afterVersion returns the payload after the version, if any. func (x xlMetaInlineData) afterVersion() []byte { if len(x) == 0 { return x } return x[1:] } // versionOK returns whether the version is ok. func (x xlMetaInlineData) versionOK() bool { if len(x) == 0 { return true } return x[0] > 0 && x[0] <= xlMetaInlineDataVer } func (x xlMetaInlineData) json() ([]byte, error) { if len(x) == 0 { return []byte("{}"), nil } if !x.versionOK() { return nil, errors.New("xlMetaInlineData: unknown version") } sz, buf, e := msgp.ReadMapHeaderBytes(x.afterVersion()) if e != nil { return nil, e } res := []byte("{") for i := uint32(0); i < sz; i++ { var key, val []byte key, buf, e = msgp.ReadMapKeyZC(buf) if e != nil { return nil, e } if len(key) == 0 { return nil, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) } // Skip data... val, buf, e = msgp.ReadBytesZC(buf) if e != nil { return nil, e } if i > 0 { res = append(res, ',') } s := fmt.Sprintf(`"%s":%d`, string(key), len(val)) res = append(res, []byte(s)...) } res = append(res, '}') return res, nil } const ( xlHeaderVersion = 2 xlMetaVersion = 1 ) func decodeXLHeaders(buf []byte) (versions int, b []byte, e error) { hdrVer, buf, e := msgp.ReadUintBytes(buf) if e != nil { return 0, buf, e } metaVer, buf, e := msgp.ReadUintBytes(buf) if e != nil { return 0, buf, e } if hdrVer > xlHeaderVersion { return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", metaVer) } if metaVer > xlMetaVersion { return 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer) } versions, buf, e = msgp.ReadIntBytes(buf) if e != nil { return 0, buf, e } if versions < 0 { return 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions) } return versions, buf, nil } // decodeVersions will decode a number of versions from a buffer // and perform a callback for each version in order, newest first. // Any non-nil error is returned. func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (e error) { var tHdr, tMeta []byte // Zero copy bytes for i := 0; i < versions; i++ { tHdr, buf, e = msgp.ReadBytesZC(buf) if e != nil { return e } tMeta, buf, e = msgp.ReadBytesZC(buf) if e != nil { return e } if e = fn(i, tHdr, tMeta); e != nil { return e } } return nil } type xlMetaV2VersionHeaderV2 struct { VersionID [16]byte ModTime int64 Signature [4]byte Type uint8 Flags uint8 } // UnmarshalMsg implements msgp.Unmarshaler func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) { var zb0001 uint32 zb0001, bts, e = msgp.ReadArrayHeaderBytes(bts) if e != nil { e = msgp.WrapError(e) return } if zb0001 != 5 { e = msgp.ArrayError{Wanted: 5, Got: zb0001} return } bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:]) if e != nil { e = msgp.WrapError(e, "VersionID") return } z.ModTime, bts, e = msgp.ReadInt64Bytes(bts) if e != nil { e = msgp.WrapError(e, "ModTime") return } bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:]) if e != nil { e = msgp.WrapError(e, "Signature") return } { var zb0002 uint8 zb0002, bts, e = msgp.ReadUint8Bytes(bts) if e != nil { e = msgp.WrapError(e, "Type") return } z.Type = zb0002 } { var zb0003 uint8 zb0003, bts, e = msgp.ReadUint8Bytes(bts) if e != nil { e = msgp.WrapError(e, "Flags") return } z.Flags = zb0003 } o = bts return } func (z xlMetaV2VersionHeaderV2) MarshalJSON() (o []byte, e error) { tmp := struct { VersionID string ModTime time.Time Signature string Type uint8 Flags uint8 }{ VersionID: hex.EncodeToString(z.VersionID[:]), ModTime: time.Unix(0, z.ModTime), Signature: hex.EncodeToString(z.Signature[:]), Type: z.Type, Flags: z.Flags, } return json.Marshal(tmp) }