MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-10-29 07:45:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

xl-meta: Add header titles (#13880)

Browse Source 
Add type for headers and create custom marshal to make 
it easier to read. Group headers and metadata.

Restore functionality that will read `xl.meta` in the current dir with no params.

Before:
```
{
  "Headers": [
    [
      "8M04bTiYRDmEMQGeAsk1yg==",
      1639150471630100400,
      "rLD1Rw==",
      1,
      6
    ],
  ],
    "Versions": [
    {
      "Type": 1,
      "V2Obj": {
        "CSumAlgo": 1,
        "DDir": "oC1Xpg4tRfW03g8o8w7Bzg==",
        "EcAlgo": 1,
        "EcBSize": 1048576,
        "EcDist": [
          7,
          8,
          1,
          2,
          3,
          4,
          5,
          6
        ],
        "EcIndex": 1,
        "EcM": 4,
        "EcN": 4,
        "ID": "8M04bTiYRDmEMQGeAsk1yg==",
        "MTime": 1639150471630100400,
        "MetaSys": {
          "x-minio-internal-inline-data": "dHJ1ZQ=="
        },
        "MetaUsr": {
          "content-type": "application/octet-stream",
          "etag": "b8252c86fad2d8937300aa92b467a3aa"
        },
        "PartASizes": [
          1000
        ],
        "PartETags": null,
        "PartNums": [
          1
        ],
        "PartSizes": [
          1000
        ],
        "Size": 1000
      }
    }
  ]
}
```

After:
```
{
  "Versions": [
    {
      "Header": {
        "Flags": 6,
        "ModTime": "2021-12-10T16:34:31.6301004+01:00",
        "Signature": "acb0f547",
        "Type": 1,
        "VersionID": "f0cd386d389844398431019e02c935ca"
      },
      "Idx": 0,
      "Metadata": {
        "Type": 1,
        "V2Obj": {
          "CSumAlgo": 1,
          "DDir": "oC1Xpg4tRfW03g8o8w7Bzg==",
          "EcAlgo": 1,
          "EcBSize": 1048576,
          "EcDist": [
            7,
            8,
            1,
            2,
            3,
            4,
            5,
            6
          ],
          "EcIndex": 1,
          "EcM": 4,
          "EcN": 4,
          "ID": "8M04bTiYRDmEMQGeAsk1yg==",
          "MTime": 1639150471630100400,
          "MetaSys": {
            "x-minio-internal-inline-data": "dHJ1ZQ=="
          },
          "MetaUsr": {
            "content-type": "application/octet-stream",
            "etag": "b8252c86fad2d8937300aa92b467a3aa"
          },
          "PartASizes": [
            1000
          ],
          "PartETags": null,
          "PartNums": [
            1
          ],
          "PartSizes": [
            1000
          ],
          "Size": 1000
        }
      }
    }
  ]
}
```
This commit is contained in:
Klaus Post 2021-12-10 15:03:25 -08:00 committed by GitHub
parent 81e43b87c2
commit 518612492c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 96 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
docs/debugging/xl-meta/main.go
View File

@ -20,6 +20,7 @@ package main
import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@ -28,6 +29,7 @@ import (
"log"
"os"
"strings"
"time"
"github.com/klauspost/compress/zip"
"github.com/minio/cli"
@ -76,10 +78,6 @@ FLAGS:
}
app.Action = func(c *cli.Context) error {
if !c.Args().Present() {
cli.ShowAppHelpAndExit(c, 1) // last argument is exit code
}
ndjson := c.Bool("ndjson")
decode := func(r io.Reader, file string) ([]byte, error) {
b, err := ioutil.ReadAll(r)
@ -128,31 +126,39 @@ FLAGS:
if err != nil {
return nil, err
}
var versions = struct {
Versions []json.RawMessage
Headers []json.RawMessage
}{
Versions: make([]json.RawMessage, nVers),
Headers: make([]json.RawMessage, nVers),
type version struct {
Idx int
Header json.RawMessage
Metadata json.RawMessage
}
var versions = make([]version, nVers)
err = decodeVersions(v, nVers, func(idx int, hdr, meta []byte) error {
var buf bytes.Buffer
if _, err := msgp.UnmarshalAsJSON(&buf, hdr); err != nil {
var header xlMetaV2VersionHeaderV2
if _, err := header.UnmarshalMsg(hdr); err != nil {
return err
}
versions.Headers[idx] = buf.Bytes()
buf = bytes.Buffer{}
b, err := header.MarshalJSON()
if err != nil {
return err
}
var buf bytes.Buffer
if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil {
return err
}
versions.Versions[idx] = buf.Bytes()
versions[idx] = version{
Idx: idx,
Header: b,
Metadata: buf.Bytes(),
}
return nil
})
if err != nil {
return nil, err
}
enc := json.NewEncoder(buf)
if err := enc.Encode(versions); err != nil {
if err := enc.Encode(struct {
Versions []version
}{Versions: versions}); err != nil {
return nil, err
}
data = b
@ -509,3 +515,77 @@ func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte)
}
return nil
}
type xlMetaV2VersionHeaderV2 struct {
VersionID [16]byte
ModTime int64
Signature [4]byte
Type uint8
Flags uint8
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zb0001 uint32
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if zb0001 != 5 {
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
return
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return
}
z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "ModTime")
return
}
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return
}
{
var zb0002 uint8
zb0002, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Type")
return
}
z.Type = zb0002
}
{
var zb0003 uint8
zb0003, bts, err = msgp.ReadUint8Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "Flags")
return
}
z.Flags = zb0003
}
o = bts
return
}
func (z xlMetaV2VersionHeaderV2) MarshalJSON() (o []byte, err error) {
tmp := struct {
VersionID string
ModTime time.Time
Signature string
Type uint8
Flags uint8
}{
VersionID: hex.EncodeToString(z.VersionID[:]),
ModTime: time.Unix(0, z.ModTime),
Signature: hex.EncodeToString(z.Signature[:]),
Type: z.Type,
Flags: z.Flags,
}
return json.Marshal(tmp)
}