minio/docs/debugging/xl-meta/main.go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/google/uuid"
	"github.com/klauspost/compress/zip"
	"github.com/klauspost/filepathx"
	"github.com/klauspost/reedsolomon"
	"github.com/minio/cli"
	"github.com/minio/highwayhash"
	"github.com/tinylib/msgp/msgp"
)

func main() {
	app := cli.NewApp()
	app.Copyright = "MinIO, Inc."
	app.Usage = "xl.meta to JSON"
	app.HideVersion = true
	app.CustomAppHelpTemplate = `NAME:
  {{.Name}} - {{.Usage}}

USAGE:
  {{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES...

Multiple files can be added. Files ending in '.zip' will be searched
for 'xl.meta' files. Wildcards are accepted: 'testdir/*.txt' will compress
all files in testdir ending with '.txt', directories can be wildcards
as well. 'testdir/*/*.txt' will match 'testdir/subdir/b.txt', double stars
means full recursive. 'testdir/**/xl.meta' will search for all xl.meta
recursively.

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
`
	isPart := regexp.MustCompile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/part\\.[0-9]+$")

	app.HideHelpCommand = true
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Usage:  "print each file as a separate line without formatting",
			Name:   "ndjson",
			Hidden: true,
		},
		cli.BoolFlag{
			Usage: "display inline data keys and sizes",
			Name:  "data",
		},
		cli.BoolFlag{
			Usage: "export inline data",
			Name:  "export",
		},
		cli.BoolFlag{
			Usage: "combine inline data",
			Name:  "combine",
		},
		cli.BoolFlag{
			Usage: "combine inline data across versions when size matches",
			Name:  "xver",
		},
	}

	app.Action = func(c *cli.Context) error {
		ndjson := c.Bool("ndjson")
		if c.Bool("data") && c.Bool("combine") {
			return errors.New("cannot combine --data and --combine")
		}
		// file / version / file
		filemap := make(map[string]map[string]string)
		foundData := make(map[string][]byte)
		partDataToVerID := make(map[string][2]string)
		var baseName string
		// versionID ->
		combineFiles := make(map[string][]string)
		decode := func(r io.Reader, file string) ([]byte, error) {
			file = strings.ReplaceAll(file, ":", "_")
			b, err := io.ReadAll(r)
			if err != nil {
				return nil, err
			}
			b, _, minor, err := checkXL2V1(b)
			if err != nil {
				return nil, err
			}
			filemap[file] = make(map[string]string)
			buf := bytes.NewBuffer(nil)
			v0 := ""
			var data xlMetaInlineData
			switch minor {
			case 0:
				_, err = msgp.CopyToJSON(buf, bytes.NewReader(b))
				if err != nil {
					return nil, err
				}
			case 1, 2:
				v, b, err := msgp.ReadBytesZC(b)
				if err != nil {
					return nil, err
				}
				if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
					// Read metadata CRC (added in v2, ignore if not found)
					b = nbuf
				}

				_, err = msgp.CopyToJSON(buf, bytes.NewReader(v))
				if err != nil {
					return nil, err
				}
				data = b
			case 3:
				v, b, err := msgp.ReadBytesZC(b)
				if err != nil {
					return nil, err
				}
				if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
					// Read metadata CRC (added in v2, ignore if not found)
					b = nbuf
				}

				hdr, v, err := decodeXLHeaders(v)
				if err != nil {
					return nil, err
				}
				type version struct {
					Idx      int
					Header   json.RawMessage
					Metadata json.RawMessage
				}
				versions := make([]version, hdr.versions)
				headerVer := hdr.headerVer
				err = decodeVersions(v, hdr.versions, func(idx int, hdr, meta []byte) error {
					var header xlMetaV2VersionHeaderV2
					if _, err := header.UnmarshalMsg(hdr, headerVer); err != nil {
						return err
					}
					b, err := header.MarshalJSON()
					if err != nil {
						return err
					}
					var buf bytes.Buffer
					if _, err := msgp.UnmarshalAsJSON(&buf, meta); err != nil {
						return err
					}
					versions[idx] = version{
						Idx:      idx,
						Header:   b,
						Metadata: buf.Bytes(),
					}
					type erasureInfo struct {
						V2Obj *struct {
							EcDist   []int
							EcIndex  int
							EcM      int
							EcN      int
							DDir     []byte
							PartNums []int
							MetaSys  struct {
								Inline []byte `json:"x-minio-internal-inline-data"`
							}
						}
					}
					var ei erasureInfo
					if err := json.Unmarshal(buf.Bytes(), &ei); err == nil && ei.V2Obj != nil {
						verID := uuid.UUID(header.VersionID).String()
						if verID == "00000000-0000-0000-0000-000000000000" {
							// If the version ID is all zeros, use the signature as version ID.
							verID = fmt.Sprintf("null/%08x", header.Signature)
							v0 = verID
						}
						idx := ei.V2Obj.EcIndex
						filemap[file][verID] = fmt.Sprintf("%s/shard-%02d-of-%02d", verID, idx, ei.V2Obj.EcN+ei.V2Obj.EcM)
						filemap[file][verID+".json"] = buf.String()
						for _, i := range ei.V2Obj.PartNums {
							if len(ei.V2Obj.MetaSys.Inline) != 0 {
								break
							}
							file := file
							dataFile := fmt.Sprintf("%s%s/part.%d", strings.TrimSuffix(file, "xl.meta"), uuid.UUID(ei.V2Obj.DDir).String(), i)
							if i > 1 {
								file = fmt.Sprintf("%s/part.%d", file, i)
								filemap[file] = make(map[string]string)
								filemap[file][verID] = fmt.Sprintf("%s/part.%d/shard-%02d-of-%02d", verID, i, idx, ei.V2Obj.EcN+ei.V2Obj.EcM)
								filemap[file][verID+".json"] = buf.String()
							}
							partDataToVerID[dataFile] = [2]string{file, verID}
						}
					} else if err != nil {
						fmt.Println("Error:", err)
					}
					return nil
				})
				if err != nil {
					return nil, err
				}
				enc := json.NewEncoder(buf)
				if err := enc.Encode(struct {
					Versions []version
				}{Versions: versions}); err != nil {
					return nil, err
				}
				data = b
			default:
				return nil, fmt.Errorf("unknown metadata version %d", minor)
			}

			if c.Bool("data") {
				b, err := data.json(true)
				if err != nil {
					return nil, err
				}
				buf = bytes.NewBuffer(b)
			}
			if c.Bool("export") {
				file := file
				if !c.Bool("combine") {
					file = strings.Map(func(r rune) rune {
						switch {
						case r >= 'a' && r <= 'z':
							return r
						case r >= 'A' && r <= 'Z':
							return r
						case r >= '0' && r <= '9':
							return r
						case strings.ContainsAny(string(r), "+=-_()!@."):
							return r
						default:
							return '_'
						}
					}, file)
				}
				if baseName == "" {
					if strings.HasSuffix(file, "/xl.meta") {
						baseName = strings.TrimSuffix(file, "/xl.meta")
						if idx := strings.LastIndexByte(baseName, '/'); idx > 0 {
							baseName = baseName[idx+1:]
						}
					}
				}
				err := data.files(func(name string, data []byte) {
					fn := fmt.Sprintf("%s-%s.data", file, name)
					if c.Bool("combine") {
						if name == "null" {
							name = v0
						}

						f := filemap[file][name]
						if f != "" {
							fn = f + ".data"
							err = os.MkdirAll(filepath.Dir(fn), os.ModePerm)
							if err != nil {
								fmt.Println("MkdirAll:", filepath.Dir(fn), err)
							}
							err = os.WriteFile(fn+".json", []byte(filemap[file][name+".json"]), os.ModePerm)
							combineFiles[name] = append(combineFiles[name], fn)
							if err != nil {
								fmt.Println("WriteFile:", err)
							}
							err = os.WriteFile(filepath.Dir(fn)+"/filename.txt", []byte(file), os.ModePerm)
							if err != nil {
								fmt.Println("combine WriteFile:", err)
							}
						}
					}
					err = os.WriteFile(fn, data, os.ModePerm)
					if err != nil {
						fmt.Println("WriteFile:", err)
					}
				})
				if err != nil {
					return nil, err
				}
			}
			if ndjson {
				return buf.Bytes(), nil
			}
			var msi map[string]interface{}
			dec := json.NewDecoder(buf)
			// Use number to preserve integers.
			dec.UseNumber()
			err = dec.Decode(&msi)
			if err != nil {
				return nil, err
			}
			b, err = json.MarshalIndent(msi, "", "  ")
			if err != nil {
				return nil, err
			}
			return b, nil
		}

		args := c.Args()
		if len(args) == 0 {
			// If no args, assume xl.meta
			args = []string{"xl.meta"}
		}
		var files []string

		for _, pattern := range args {
			if pattern == "-" {
				files = append(files, pattern)
				continue
			}
			found, err := filepathx.Glob(pattern)
			if err != nil {
				return err
			}
			if len(found) == 0 {
				return fmt.Errorf("unable to find file %v", pattern)
			}
			files = append(files, found...)
		}
		if len(files) == 0 {
			return fmt.Errorf("no files found")
		}
		if len(files) > 1 || strings.HasSuffix(files[0], ".zip") {
			ndjson = true
		}

		toPrint := make([]string, 0, 16)
		for _, file := range files {
			var r io.Reader
			var sz int64
			switch file {
			case "-":
				r = os.Stdin
			default:
				f, err := os.Open(file)
				if err != nil {
					return err
				}
				if st, err := f.Stat(); err == nil {
					sz = st.Size()
				}
				defer f.Close()
				r = f
			}
			if strings.HasSuffix(file, ".zip") {
				zr, err := zip.NewReader(r.(io.ReaderAt), sz)
				if err != nil {
					return err
				}
				for _, file := range zr.File {
					if file.FileInfo().IsDir() {
						continue
					}
					if strings.HasSuffix(file.Name, "xl.meta") {
						r, err := file.Open()
						if err != nil {
							return err
						}
						// Quote string...
						b, _ := json.Marshal(file.Name)
						b2, err := decode(r, file.Name)
						if err != nil {
							return err
						}
						var tmp map[string]interface{}
						if err := json.Unmarshal(b2, &tmp); err == nil {
							if b3, err := json.Marshal(tmp); err == nil {
								b2 = b3
							}
						}
						toPrint = append(toPrint, fmt.Sprintf("\t%s: %s", string(b), string(b2)))
					} else if c.Bool("combine") && isPart.MatchString(file.Name) {
						// name := isPart.FindString(file.Name)
						name := strings.ReplaceAll(file.Name, ":", "_")
						r, err := file.Open()
						if err != nil {
							return err
						}
						all, err := io.ReadAll(r)
						if err != nil {
							return err
						}
						foundData[name] = all
					}
				}
			} else {
				b0 := ""
				if ndjson {
					b, _ := json.Marshal(file)
					b0 = fmt.Sprintf("%s: ", string(b))
				}
				b, err := decode(r, file)
				if err != nil {
					return err
				}
				b = bytes.TrimSpace(b)
				if !ndjson {
					b = bytes.TrimFunc(b, func(r rune) bool {
						return r == '{' || r == '}' || r == '\n' || r == '\r'
					})
				}

				toPrint = append(toPrint, fmt.Sprintf("%s%s", b0, string(b)))
			}
		}
		sort.Strings(toPrint)
		fmt.Printf("{\n%s\n}\n", strings.Join(toPrint, ",\n"))
		if c.Bool("combine") {
			for partName, data := range foundData {
				if verid := partDataToVerID[partName]; verid != [2]string{} {
					file := verid[0]
					name := verid[1]
					f := filemap[file][name]
					fn := fmt.Sprintf("%s-%s.data", file, name)
					if f != "" {
						fn = f + ".data"
						err := os.MkdirAll(filepath.Dir(fn), os.ModePerm)
						if err != nil {
							fmt.Println("MkdirAll:", filepath.Dir(fn), err)
						}
						err = os.WriteFile(fn+".json", []byte(filemap[file][name+".json"]), os.ModePerm)
						combineFiles[name] = append(combineFiles[name], fn)
						if err != nil {
							fmt.Println("WriteFile:", err)
						}
						err = os.WriteFile(filepath.Dir(fn)+"/filename.txt", []byte(file), os.ModePerm)
						if err != nil {
							fmt.Println("combine WriteFile:", err)
						}
						fmt.Println("Remapped", partName, "to", fn)
					}
					delete(partDataToVerID, partName)
					err := os.WriteFile(fn, data, os.ModePerm)
					if err != nil {
						fmt.Println("WriteFile:", err)
					}
				}
			}
			if len(partDataToVerID) > 0 {
				fmt.Println("MISSING PART FILES:")
				for k := range partDataToVerID {
					fmt.Println(k)
				}
				fmt.Println("END MISSING PART FILES")
			}
		}

		if len(combineFiles) > 0 {
			if c.Bool("xver") {
				if err := combineCrossVer(combineFiles, baseName); err != nil {
					fmt.Println("ERROR:", err)
				}
			} else {
				for k, v := range combineFiles {
					if err := combine(v, k+"-"+baseName); err != nil {
						fmt.Println("ERROR:", err)
					}
				}
			}
		}

		return nil
	}
	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

var (
	// XL header specifies the format
	xlHeader = [4]byte{'X', 'L', '2', ' '}

	// Current version being written.
	xlVersionCurrent [4]byte
)

const (
	// Breaking changes.
	// Newer versions cannot be read by older software.
	// This will prevent downgrades to incompatible versions.
	xlVersionMajor = 1

	// Non breaking changes.
	// Bumping this is informational, but should be done
	// if any change is made to the data stored, bumping this
	// will allow to detect the exact version later.
	xlVersionMinor = 1
)

func init() {
	binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor)
	binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor)
}

// checkXL2V1 will check if the metadata has correct header and is a known major version.
// The remaining payload and versions are returned.
func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) {
	if len(buf) <= 8 {
		return payload, 0, 0, fmt.Errorf("xlMeta: no data")
	}

	if !bytes.Equal(buf[:4], xlHeader[:]) {
		return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4])
	}

	if bytes.Equal(buf[4:8], []byte("1   ")) {
		// Set as 1,0.
		major, minor = 1, 0
	} else {
		major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8])
	}
	if major > xlVersionMajor {
		return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major)
	}

	return buf[8:], major, minor, nil
}

const xlMetaInlineDataVer = 1

type xlMetaInlineData []byte

// afterVersion returns the payload after the version, if any.
func (x xlMetaInlineData) afterVersion() []byte {
	if len(x) == 0 {
		return x
	}
	return x[1:]
}

// versionOK returns whether the version is ok.
func (x xlMetaInlineData) versionOK() bool {
	if len(x) == 0 {
		return true
	}
	return x[0] > 0 && x[0] <= xlMetaInlineDataVer
}

func (x xlMetaInlineData) json(value bool) ([]byte, error) {
	if len(x) == 0 {
		return []byte("{}"), nil
	}
	if !x.versionOK() {
		return nil, errors.New("xlMetaInlineData: unknown version")
	}
	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
	if err != nil {
		return nil, err
	}
	res := []byte("{")

	for i := uint32(0); i < sz; i++ {
		var key, val []byte
		key, buf, err = msgp.ReadMapKeyZC(buf)
		if err != nil {
			return nil, err
		}
		if len(key) == 0 {
			return nil, fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
		}
		// Skip data...
		val, buf, err = msgp.ReadBytesZC(buf)
		if err != nil {
			return nil, err
		}
		if i > 0 {
			res = append(res, ',')
		}
		s := fmt.Sprintf(`"%s": {"bytes": %d`, string(key), len(val))
		// Check bitrot... We should only ever have one block...
		if len(val) >= 32 {
			want := val[:32]
			data := val[32:]
			const magicHighwayHash256Key = "\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0"

			hh, _ := highwayhash.New([]byte(magicHighwayHash256Key))
			hh.Write(data)
			got := hh.Sum(nil)
			if bytes.Equal(want, got) {
				s += ", \"bitrot_valid\": true"
			} else {
				s += ", \"bitrot_valid\": false"
			}
			if value {
				if utf8.Valid(data) {
					// Encode as JSON string.
					b, err := json.Marshal(string(data))
					if err == nil {
						s += `, "data_string": ` + string(b)
					}
				}
				// Base64 encode.
				s += `, "data_base64": "` + base64.StdEncoding.EncodeToString(data) + `"`
			}
			s += "}"
		}
		res = append(res, []byte(s)...)
	}
	res = append(res, '}')
	return res, nil
}

// files returns files as callback.
func (x xlMetaInlineData) files(fn func(name string, data []byte)) error {
	if len(x) == 0 {
		return nil
	}
	if !x.versionOK() {
		return errors.New("xlMetaInlineData: unknown version")
	}

	sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion())
	if err != nil {
		return err
	}

	for i := uint32(0); i < sz; i++ {
		var key, val []byte
		key, buf, err = msgp.ReadMapKeyZC(buf)
		if err != nil {
			return err
		}
		if len(key) == 0 {
			return fmt.Errorf("xlMetaInlineData: key %d is length 0", i)
		}
		// Read data...
		val, buf, err = msgp.ReadBytesZC(buf)
		if err != nil {
			return err
		}
		// Call back.
		fn(string(key), val)
	}
	return nil
}

const (
	xlHeaderVersion = 3
	xlMetaVersion   = 2
)

type xlHeaders struct {
	versions           int
	headerVer, metaVer uint
}

func decodeXLHeaders(buf []byte) (x xlHeaders, b []byte, err error) {
	x.headerVer, buf, err = msgp.ReadUintBytes(buf)
	if err != nil {
		return x, buf, err
	}
	x.metaVer, buf, err = msgp.ReadUintBytes(buf)
	if err != nil {
		return x, buf, err
	}
	if x.headerVer > xlHeaderVersion {
		return x, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", x.headerVer)
	}
	if x.metaVer > xlMetaVersion {
		return x, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", x.metaVer)
	}
	x.versions, buf, err = msgp.ReadIntBytes(buf)
	if err != nil {
		return x, buf, err
	}
	if x.versions < 0 {
		return x, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", x.versions)
	}
	return x, buf, nil
}

// decodeVersions will decode a number of versions from a buffer
// and perform a callback for each version in order, newest first.
// Any non-nil error is returned.
func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) {
	var tHdr, tMeta []byte // Zero copy bytes
	for i := 0; i < versions; i++ {
		tHdr, buf, err = msgp.ReadBytesZC(buf)
		if err != nil {
			return err
		}
		tMeta, buf, err = msgp.ReadBytesZC(buf)
		if err != nil {
			return err
		}
		if err = fn(i, tHdr, tMeta); err != nil {
			return err
		}
	}
	return nil
}

type xlMetaV2VersionHeaderV2 struct {
	VersionID [16]byte
	ModTime   int64
	Signature [4]byte
	Type      uint8
	Flags     uint8
	EcN, EcM  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
}

// UnmarshalMsg implements msgp.Unmarshaler
func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byte, err error) {
	var zb0001 uint32
	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
	if err != nil {
		err = msgp.WrapError(err)
		return
	}
	want := uint32(5)
	if hdrVer > 2 {
		want += 2
	}
	if zb0001 != want {
		err = msgp.ArrayError{Wanted: want, Got: zb0001}
		return
	}
	bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
	if err != nil {
		err = msgp.WrapError(err, "VersionID")
		return
	}
	z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
	if err != nil {
		err = msgp.WrapError(err, "ModTime")
		return
	}
	bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
	if err != nil {
		err = msgp.WrapError(err, "Signature")
		return
	}
	{
		var zb0002 uint8
		zb0002, bts, err = msgp.ReadUint8Bytes(bts)
		if err != nil {
			err = msgp.WrapError(err, "Type")
			return
		}
		z.Type = zb0002
	}
	{
		var zb0003 uint8
		zb0003, bts, err = msgp.ReadUint8Bytes(bts)
		if err != nil {
			err = msgp.WrapError(err, "Flags")
			return
		}
		z.Flags = zb0003
	}
	if hdrVer > 2 {
		// Version 3 has EcM and EcN
		{
			var zb0004 uint8
			zb0004, bts, err = msgp.ReadUint8Bytes(bts)
			if err != nil {
				err = msgp.WrapError(err, "EcN")
				return
			}
			z.EcN = zb0004
		}
		{
			var zb0005 uint8
			zb0005, bts, err = msgp.ReadUint8Bytes(bts)
			if err != nil {
				err = msgp.WrapError(err, "EcM")
				return
			}
			z.EcM = zb0005
		}
	}
	o = bts
	return
}

func (z xlMetaV2VersionHeaderV2) MarshalJSON() (o []byte, err error) {
	tmp := struct {
		VersionID string
		ModTime   time.Time
		Signature string
		Type      uint8
		Flags     uint8
		EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
	}{
		VersionID: hex.EncodeToString(z.VersionID[:]),
		ModTime:   time.Unix(0, z.ModTime),
		Signature: hex.EncodeToString(z.Signature[:]),
		Type:      z.Type,
		Flags:     z.Flags,
		EcM:       z.EcM,
		EcN:       z.EcN,
	}
	return json.Marshal(tmp)
}

type mappedData struct {
	mapped, filled             []byte
	size, shards, data, parity int
	parityData                 map[int]map[int][]byte
	blockOffset                int // Offset in bytes to start of block.
	blocks                     int // 0 = one block.
	objSize, partSize          int
	wantMD5                    string
}

func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
	var m mappedData
	sort.Strings(files)
	m.parityData = make(map[int]map[int][]byte)
	for _, file := range files {
		meta, err := os.ReadFile(file + ".json")
		if err != nil {
			return nil, err
		}
		type erasureInfo struct {
			V2Obj *struct {
				EcDist    []int
				EcIndex   int
				EcM       int
				EcN       int
				Size      int
				EcBSize   int
				PartNums  []int
				PartSizes []int
				MetaUsr   struct {
					Etag string `json:"etag"`
				}
			}
		}
		var ei erasureInfo
		var idx int
		if err := json.Unmarshal(meta, &ei); err == nil && ei.V2Obj != nil {
			if m.objSize == 0 {
				m.objSize = ei.V2Obj.Size
			}
			m.data = ei.V2Obj.EcM
			m.parity = ei.V2Obj.EcN
			if len(ei.V2Obj.PartNums) == 1 && !strings.ContainsRune(ei.V2Obj.MetaUsr.Etag, '-') {
				m.wantMD5 = ei.V2Obj.MetaUsr.Etag
			}
			if m.shards == 0 {
				m.shards = m.data + m.parity
			}
			idx = ei.V2Obj.EcIndex - 1
			fmt.Println("Read shard", ei.V2Obj.EcIndex, fmt.Sprintf("(%s)", file))
			if ei.V2Obj.Size != m.objSize {
				return nil, fmt.Errorf("size mismatch. Meta size: %d, Prev: %d", ei.V2Obj.Size, m.objSize)
			}
			for i, s := range ei.V2Obj.PartNums {
				if s == partNum {
					m.size = ei.V2Obj.PartSizes[i]
					m.partSize = ei.V2Obj.PartSizes[i]
					break
				}
			}
		} else {
			return nil, err
		}

		offset := ei.V2Obj.EcBSize * blockNum
		if offset >= m.size {
			return nil, fmt.Errorf("block %d out of range. offset %d > size %d", blockNum, offset, m.size)
		}
		m.blockOffset = offset
		m.blocks = (m.size + ei.V2Obj.EcBSize - 1) / ei.V2Obj.EcBSize
		if m.blocks > 0 {
			m.blocks--
		}
		if blockNum < m.blocks {
			m.size = ei.V2Obj.EcBSize
		} else {
			m.size -= offset
		}

		b, err := os.ReadFile(file)
		if err != nil {
			return nil, err
		}
		if len(b) < 32 {
			return nil, fmt.Errorf("file %s too short", file)
		}

		// Extract block data.
		ssz := shardSize(ei.V2Obj.EcBSize, ei.V2Obj.EcM)
		b, err = bitrot(b, blockNum*ssz, ssz)
		if err != nil {
			return nil, err
		}

		if m.mapped == nil {
			m.mapped = make([]byte, m.size)
			m.filled = make([]byte, m.size)
		}

		set := m.parityData[m.data]
		if set == nil {
			set = make(map[int][]byte)
		}
		set[idx] = b
		m.parityData[m.data] = set

		// Combine
		start := len(b) * idx
		if start >= len(m.mapped) {
			continue
		}
		fmt.Println("Block data size:", m.size, "Shard size", ssz, "Got Shard:", len(b), "Bitrot ok", "Start", start, "End", start+len(b))
		copy(m.mapped[start:], b)
		for j := range b {
			if j+start >= len(m.filled) {
				break
			}
			m.filled[j+start] = 1
		}
	}
	return &m, nil
}

func combine(files []string, out string) error {
	fmt.Printf("Attempting to combine version %q.\n", out)
	m, err := readAndMap(files, 1, 0)
	if err != nil {
		return err
	}
	if m.blocks > 0 {
		// TODO: Support multiple blocks. For now use -xver.
		return fmt.Errorf("multiple blocks found, only one block supported. Try with -xver")
	}
	lastValid := 0
	missing := 0
	for i := range m.filled {
		if m.filled[i] == 1 {
			lastValid = i
		} else {
			missing++
		}
	}
	if missing > 0 && len(m.parityData) > 0 {
		fmt.Println("Attempting to reconstruct using parity sets:")
		for k, v := range m.parityData {
			if missing == 0 {
				break
			}
			fmt.Println("* Setup: Data shards:", k, "- Parity blocks:", m.shards-k)
			rs, err := reedsolomon.New(k, m.shards-k)
			if err != nil {
				return err
			}
			split, err := rs.Split(m.mapped)
			if err != nil {
				return err
			}
			splitFilled, err := rs.Split(m.filled)
			if err != nil {
				return err
			}
			ok := len(splitFilled)
			for i, sh := range splitFilled {
				for _, v := range sh {
					if v == 0 {
						split[i] = nil
						ok--
						break
					}
				}
			}
			hasParity := 0
			for idx, sh := range v {
				split[idx] = sh
				if idx >= k && len(sh) > 0 {
					hasParity++
				}
			}
			fmt.Printf("Have %d complete remapped data shards and %d complete parity shards. ", ok, hasParity)

			if err := rs.ReconstructData(split); err == nil {
				fmt.Println("Could reconstruct completely")
				for i, data := range split[:k] {
					start := i * len(data)
					copy(m.mapped[start:], data)
				}
				lastValid = m.size - 1
				missing = 0
			} else {
				fmt.Println("Could NOT reconstruct:", err)
			}
		}
	}
	if lastValid == 0 {
		return errors.New("no valid data found")
	}
	if missing > 0 {
		fmt.Println(missing, "bytes missing. Truncating", len(m.filled)-lastValid-1, "from end.")
		out += ".incomplete"
	} else {
		fmt.Println("No bytes missing.")
		out += ".complete"
	}
	m.mapped = m.mapped[:lastValid+1]
	err = os.WriteFile(out, m.mapped, os.ModePerm)
	if err != nil {
		return err
	}
	fmt.Println("Wrote output to", out)
	return nil
}

func combineCrossVer(all map[string][]string, baseName string) error {
	names := make([][]string, 0)
	/// part, verID, file
	files := make([]map[string][]string, 0)
	partNums := make(map[int]int)
	for k, v := range all {
		for _, file := range v {
			part := getPartNum(file)
			partIdx, ok := partNums[part]
			if !ok {
				partIdx = len(names)
				partNums[part] = partIdx
				names = append(names, nil)
				files = append(files, make(map[string][]string))
			}
			names[partIdx] = append(names[partIdx], k)
			files[partIdx][k] = append(files[partIdx][k], file)
		}
	}
	if len(files) == 0 {
		return nil
	}
	for part, partIdx := range partNums {
		if len(files[partIdx]) == 0 {
			continue
		}
		var wantMD5 string
		exportedSizes := make(map[int]bool)
		// block -> data
		combineSharedBlocks := make(map[int][]byte)
		combineFilledBlocks := make(map[int][]byte)
	nextFile:
		for key, file := range files[partIdx] {
			fmt.Println("Reading base version", file[0], "part", part)
			var combined []byte
			var missingAll int
			var lastValidAll int

			attempt := 0
			for block := 0; ; block++ {
				combineFilled := combineFilledBlocks[block]
				combineShared := combineSharedBlocks[block]
			nextAttempt:
				fmt.Printf("Block %d, Base version %q. Part %d. Files %d\n", block+1, key, part, len(file))
				m, err := readAndMap(file, part, block)
				if err != nil {
					return err
				}
				if exportedSizes[m.objSize] {
					fmt.Println("Skipping version", key, "as it has already been exported.")
					continue nextFile
				}
				addedFiles := 0
			compareFile:
				for otherKey, other := range files[partIdx] {
					addedFiles++
					if attempt > 0 && len(m.filled) == len(combineFilled) {
						fmt.Println("Merging previous global data")
						filled := 0
						missing := 0
						for i, v := range combineFilled {
							if v == 1 {
								m.filled[i] = 1
								m.mapped[i] = combineShared[i]
								filled++
							} else {
								missing++
							}
						}
						fmt.Println("Missing", missing, "bytes. Filled", filled, "bytes.")
						break
					}
					if key == otherKey {
						continue
					}

					otherPart := getPartNum(other[0])
					if part != otherPart {
						fmt.Println("part ", part, " != other part", otherPart, other[0])
						continue
					}
					// fmt.Println("part ", part, "other part", otherPart, other[0])
					fmt.Printf("Reading version %q Part %d.\n", otherKey, otherPart)
					// os.Exit(0)
					otherM, err := readAndMap(other, part, block)
					if err != nil {
						fmt.Println(err)
						continue
					}
					if m.objSize != otherM.objSize {
						continue
					}

					// If data+parity matches, combine.
					if m.parity == otherM.parity && m.data == otherM.data {
						for k, v := range m.parityData {
							if otherM.parityData[k] == nil {
								continue
							}
							for i, data := range v {
								if data != nil || otherM.parityData[k][i] == nil {
									continue
								}
								m.parityData[k][i] = otherM.parityData[k][i]
							}
						}
					}

					var ok int
					for i, filled := range otherM.filled[:m.size] {
						if filled == 1 && m.filled[i] == 1 {
							if m.mapped[i] != otherM.mapped[i] {
								fmt.Println("Data mismatch at byte", i, "-  Disregarding version", otherKey)
								continue compareFile
							}
							ok++
						}
					}

					fmt.Printf("Data overlaps (%d bytes). Combining with %q.\n", ok, otherKey)
					for i := range otherM.filled {
						if otherM.filled[i] == 1 {
							m.filled[i] = 1
							m.mapped[i] = otherM.mapped[i]
						}
					}
				}

				lastValid := 0
				missing := 0
				for i := range m.filled {
					if m.filled[i] == 1 {
						lastValid = i
					} else {
						missing++
					}
				}
				if missing > 0 && len(m.parityData) > 0 {
					fmt.Println("Attempting to reconstruct using parity sets:")
					for k, v := range m.parityData {
						if missing == 0 {
							break
						}
						fmt.Println("* Setup: Data shards:", k, "- Parity blocks:", m.shards-k)
						rs, err := reedsolomon.New(k, m.shards-k)
						if err != nil {
							return err
						}
						splitData, err := rs.Split(m.mapped)
						if err != nil {
							return err
						}
						splitFilled, err := rs.Split(m.filled)
						if err != nil {
							return err
						}
						// Fill padding...
						padding := len(splitFilled[0])*k - len(m.filled)
						for i := 0; i < padding; i++ {
							arr := splitFilled[k-1]
							arr[len(arr)-i-1] = 1
						}

						hasParity := 0
						parityOK := make([]bool, m.shards)
						for idx, sh := range v {
							splitData[idx] = sh
							if idx >= k && len(sh) > 0 {
								parityOK[idx] = true
								hasParity++
								for i := range splitFilled[idx] {
									splitFilled[idx][i] = 1
								}
							}
						}

						splitDataShards := make([]byte, len(splitFilled[0]))
						for _, sh := range splitFilled {
							for i, v := range sh {
								splitDataShards[i] += v
							}
						}
						var hist [256]int
						for _, v := range splitDataShards {
							hist[v]++
						}

						for _, v := range hist[m.data-hasParity : m.shards] {
							if attempt > 0 {
								break
							}
							if v == 0 {
								continue
							}
							for i, v := range hist[:m.shards] {
								if v > 0 {
									if i < m.data {
										fmt.Println("- Shards:", i, "of", m.data, "Bytes:", v, "Missing: ", v*(m.data-i+hasParity))
									} else {
										fmt.Println("+ Shards:", i, "of", m.data, "Bytes:", v, "Recovering: ", v*(m.data-i+hasParity))
									}
								}
							}
							fmt.Println("Attempting to reconstruct with partial shards")
							offset := 0
							startOffset := 0
							shardConfig := make([]byte, k)
							reconstructAbleConfig := false
							shards := make([][]byte, m.shards)
							for i := range shards {
								shards[i] = make([]byte, 0, len(splitData[0]))
							}
							for offset < len(splitDataShards) {
								newConfig := false
								for shardIdx, shard := range splitFilled[:k] {
									if shardConfig[shardIdx] != shard[offset] {
										newConfig = true
										break
									}
								}
								if newConfig {
									if offset > startOffset && reconstructAbleConfig {
										reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
									}
									// Update to new config and add current
									valid := 0
									for shardIdx, shard := range splitFilled[:k] {
										shardConfig[shardIdx] = shard[offset]
										valid += int(shard[offset])
										if shard[offset] == 0 {
											shards[shardIdx] = shards[shardIdx][:0]
										} else {
											shards[shardIdx] = append(shards[shardIdx][:0], splitData[shardIdx][offset])
										}
									}
									reconstructAbleConfig = valid >= m.data-hasParity && valid < m.data
									startOffset = offset
									offset++
									continue
								}
								for shardIdx, ok := range shardConfig {
									if ok != 0 {
										shards[shardIdx] = append(shards[shardIdx], splitData[shardIdx][offset])
									}
								}
								offset++
							}
							if offset > startOffset && reconstructAbleConfig {
								reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
							}

							var buf bytes.Buffer
							if err := rs.Join(&buf, splitFilled, m.size); err == nil {
								m.filled = buf.Bytes()
							}
							buf = bytes.Buffer{}
							if err := rs.Join(&buf, splitData, m.size); err == nil {
								m.mapped = buf.Bytes()
							}
							for i, v := range m.filled {
								if v == 0 {
									m.mapped[i] = 0
								}
							}
							break
						}
						ok := k
						for i, sh := range splitFilled {
							for j, v := range sh {
								if v == 0 {
									splitData[i] = nil
									if i < k {
										fmt.Println("Shard", i, "is missing data from offset", i*len(sh)+j)
										ok--
									}
									break
								}
							}
						}

						missing = 0
						lastValid = 0
						for i := range m.filled {
							if m.filled[i] == 1 {
								lastValid = i
							} else {
								missing++
							}
						}
						fmt.Printf("Have %d complete remapped data shards and %d complete parity shards (%d bytes missing). ", ok, hasParity, missing)

						if err := rs.ReconstructData(splitData); err == nil {
							fmt.Println("Could reconstruct completely.")
							for i, data := range splitData[:k] {
								start := i * len(data)
								copy(m.mapped[start:], data)
							}
							lastValid = m.size - 1
							missing = 0
							attempt = 2
							wantMD5 = m.wantMD5
						} else {
							fmt.Println("Could NOT reconstruct:", err, " - Need", m.data, "shards.")
							if attempt == 0 {
								if len(combineShared) == 0 {
									combineShared = make([]byte, len(m.mapped))
									combineFilled = make([]byte, len(m.filled))
								}
								for i := range m.filled {
									if m.filled[i] == 1 && combineFilled[i] == 0 {
										combineShared[i] = m.mapped[i]
										combineFilled[i] = 1
									}
								}
								combineFilledBlocks[block] = combineFilled
								combineSharedBlocks[block] = combineShared
								fmt.Println("Retrying with merged data")
								if addedFiles >= len(files[partIdx]) {
									attempt++
									goto nextAttempt
								}
							}
						}
					}
				}
				if m.blockOffset != len(combined) {
					return fmt.Errorf("Block offset mismatch. Expected %d got %d", m.blockOffset, len(combined))
				}
				combined = append(combined, m.mapped[:m.size]...)
				missingAll += missing
				if lastValid > 0 {
					lastValidAll = lastValid + m.blockOffset
				}
				if m.blocks == block {
					if len(combined) != m.partSize {
						fmt.Println("Combined size mismatch. Expected", m.partSize, "got", len(combined))
					}
					fmt.Println("Reached block", block+1, "of", m.blocks+1, "for", key, "Done.")
					break
				}
			}
			if lastValidAll == 0 {
				return errors.New("no valid data found")
			}
			out := fmt.Sprintf("%s-%s.%05d", key, baseName, part)
			if len(files) == 1 {
				out = fmt.Sprintf("%s-%s", key, baseName)
			}
			if missingAll > 0 {
				out += ".incomplete"
				fmt.Println(missingAll, "bytes missing.")
			} else {
				if wantMD5 != "" {
					sum := md5.Sum(combined)
					gotMD5 := hex.EncodeToString(sum[:])
					if gotMD5 != wantMD5 {
						fmt.Println("MD5 mismatch. Expected", wantMD5, "got", gotMD5)
						out += ".mismatch"
					} else {
						fmt.Println("MD5 verified.")
						out = fmt.Sprintf("verified/%s", baseName)
					}
				} else {
					out = fmt.Sprintf("complete/%s.%05d", baseName, part)
					fmt.Println("No bytes missing.")
				}
			}
			if missingAll == 0 {
				exportedSizes[len(combined)] = true
			}
			err := os.MkdirAll(filepath.Dir(out), os.ModePerm)
			if err != nil {
				return err
			}
			err = os.WriteFile(out, combined, os.ModePerm)
			if err != nil {
				return err
			}
			fmt.Println("Wrote output to", out)
		}
	}
	return nil
}

func reconPartial(shards [][]byte, k int, parityOK []bool, splitData [][]byte, startOffset int, offset int, rs reedsolomon.Encoder, shardConfig []byte, splitFilled [][]byte) {
	// Add parity
	for i := range shards[k:] {
		shards[i+k] = nil
		if parityOK[i+k] {
			shards[i+k] = splitData[i+k][startOffset:offset]
		}
	}
	// Reconstruct with current config.
	if err := rs.ReconstructData(shards); err != nil {
		panic(fmt.Sprintln("Internal error, could NOT partially reconstruct:", err))
	}
	// Copy reconstructed data back.
	verified := 0
	reconstructed := 0
	for shardsIdx, ok := range shardConfig {
		if ok == 0 {
			copy(splitData[shardsIdx][startOffset:], shards[shardsIdx])
			for i := range shards[shardsIdx] {
				if splitFilled[shardsIdx][startOffset+i] == 1 {
					fmt.Println("Internal error: Found filled data at", startOffset+i)
				}
				splitFilled[shardsIdx][startOffset+i] = 1
			}
			reconstructed += len(shards[shardsIdx])
		} else {
			for i := range shards[shardsIdx] {
				if splitFilled[shardsIdx][startOffset+i] == 0 {
					fmt.Println("Internal error: Expected filled data at", startOffset+i)
				}
				if splitData[shardsIdx][startOffset+i] != shards[shardsIdx][i] {
					fmt.Println("Internal error: Mismatch at", startOffset+i)
				}
				verified++
			}
		}
	}
	fmt.Println("Reconstructed", reconstructed, "bytes and verified", verified, "bytes of partial shard with config", shardConfig)
}

// bitrot returns a shard beginning at startOffset after doing bitrot checks.
func bitrot(val []byte, startOffset, shardSize int) ([]byte, error) {
	var res []byte
	for len(val) >= 32 {
		want := val[:32]
		data := val[32:]
		if len(data) > shardSize {
			data = data[:shardSize]
		}

		const magicHighwayHash256Key = "\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0"

		hh, _ := highwayhash.New([]byte(magicHighwayHash256Key))
		hh.Write(data)
		if !bytes.Equal(want, hh.Sum(nil)) {
			return res, fmt.Errorf("bitrot detected")
		}
		res = append(res, data...)
		val = val[32+len(data):]
		if len(res) > startOffset {
			return res[startOffset:], nil
		}
	}
	return res, fmt.Errorf("bitrot: data too short to get block. len(res)=%d, startOffset=%d", len(res), startOffset)
}

// shardSize returns the shard size for a given block size and data blocks.
func shardSize(blockSize, dataBlocks int) (sz int) {
	if dataBlocks == 0 {
		// do nothing on invalid input
		return
	}
	// Make denominator positive
	if dataBlocks < 0 {
		blockSize = -blockSize
		dataBlocks = -dataBlocks
	}
	sz = blockSize / dataBlocks
	if blockSize > 0 && blockSize%dataBlocks != 0 {
		sz++
	}
	return
}

var rePartNum = regexp.MustCompile("/part\\.([0-9]+)/")

func getPartNum(s string) int {
	if m := rePartNum.FindStringSubmatch(s); len(m) > 1 {
		n, _ := strconv.Atoi(m[1])
		return n
	}
	return 1
}