diff --git a/cmd/format-fs.go b/cmd/format-fs.go new file mode 100644 index 000000000..63227a4cb --- /dev/null +++ b/cmd/format-fs.go @@ -0,0 +1,226 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "os" + "time" + + errors2 "github.com/minio/minio/pkg/errors" + "github.com/minio/minio/pkg/lock" +) + +// FS format version strings. +const ( + formatBackendFS = "fs" + formatFSVersionV1 = "1" +) + +// formatFSV1 - structure holds format version '1'. +type formatFSV1 struct { + formatMetaV1 + FS struct { + Version string `json:"version"` + } `json:"fs"` +} + +// Used to detect the version of "fs" format. +type formatFSVersionDetect struct { + FS struct { + Version string `json:"version"` + } `json:"fs"` +} + +// Returns the latest "fs" format. +func newFormatFSV1() (format *formatFSV1) { + f := &formatFSV1{} + f.Version = formatMetaVersionV1 + f.Format = formatBackendFS + f.FS.Version = formatFSVersionV1 + return f +} + +// Save to fs format.json +func formatFSSave(f *os.File, data interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return errors2.Trace(err) + } + if err = f.Truncate(0); err != nil { + return errors2.Trace(err) + } + if _, err = f.Seek(0, io.SeekStart); err != nil { + return err + } + _, err = f.Write(b) + if err != nil { + return errors2.Trace(err) + } + return nil +} + +// Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change. +// We do not use this function in XL to get the format as the file is not fcntl-locked on XL. +func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) { + format := &formatMetaV1{} + if err := jsonLoadFromSeeker(r, format); err != nil { + return "", err + } + if format.Version == formatMetaVersionV1 { + return format.Format, nil + } + return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, format.Version) +} + +// Returns formatFS.FS.Version +func formatFSGetVersion(r io.ReadSeeker) (string, error) { + format := &formatFSVersionDetect{} + if err := jsonLoadFromSeeker(r, format); err != nil { + return "", err + } + return format.FS.Version, nil +} + +// Migrate the "fs" backend. +// Migration should happen when formatFSV1.FS.Version changes. This version +// can change when there is a change to the struct formatFSV1.FS or if there +// is any change in the backend file system tree structure. +func formatFSMigrate(wlk *lock.LockedFile) error { + // Add any migration code here in case we bump format.FS.Version + + // Make sure that the version is what we expect after the migration. + version, err := formatFSGetVersion(wlk) + if err != nil { + return err + } + if version != formatFSVersionV1 { + return fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV1, version) + } + return nil +} + +// Creates a new format.json if unformatted. +func createFormatFS(fsFormatPath string) error { + // Attempt a write lock on formatConfigFile `format.json` + // file stored in minioMetaBucket(.minio.sys) directory. + lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return errors2.Trace(err) + } + // Close the locked file upon return. + defer lk.Close() + + fi, err := lk.Stat() + if err != nil { + return errors2.Trace(err) + } + if fi.Size() != 0 { + // format.json already got created because of another minio process's createFormatFS() + return nil + } + + return formatFSSave(lk.File, newFormatFSV1()) +} + +// This function returns a read-locked format.json reference to the caller. +// The file descriptor should be kept open throughout the life +// of the process so that another minio process does not try to +// migrate the backend when we are actively working on the backend. +func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { + fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile) + // Any read on format.json should be done with read-lock. + // Any write on format.json should be done with write-lock. + for { + isEmpty := false + rlk, err := lock.RLockedOpenFile(fsFormatPath) + if err == nil { + // format.json can be empty in a rare condition when another + // minio process just created the file but could not hold lock + // and write to it. + var fi os.FileInfo + fi, err = rlk.Stat() + if err != nil { + return nil, errors2.Trace(err) + } + isEmpty = fi.Size() == 0 + } + if os.IsNotExist(err) || isEmpty { + if err == nil { + rlk.Close() + } + // Fresh disk - create format.json + err = createFormatFS(fsFormatPath) + if err == lock.ErrAlreadyLocked { + // Lock already present, sleep and attempt again. + // Can happen in a rare situation when a parallel minio process + // holds the lock and creates format.json + time.Sleep(100 * time.Millisecond) + continue + } + if err != nil { + return nil, errors2.Trace(err) + } + // After successfully creating format.json try to hold a read-lock on + // the file. + continue + } + if err != nil { + return nil, errors2.Trace(err) + } + + formatBackend, err := formatMetaGetFormatBackendFS(rlk) + if err != nil { + return nil, errors2.Trace(err) + } + if formatBackend != formatBackendFS { + return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend) + } + version, err := formatFSGetVersion(rlk) + if err != nil { + return nil, err + } + if version != formatFSVersionV1 { + // Format needs migration + rlk.Close() + // Hold write lock during migration so that we do not disturb any + // minio processes running in parallel. + wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0) + if err == lock.ErrAlreadyLocked { + // Lock already present, sleep and attempt again. + wlk.Close() + time.Sleep(100 * time.Millisecond) + continue + } + if err != nil { + return nil, err + } + err = formatFSMigrate(wlk) + wlk.Close() + if err != nil { + // Migration failed, bail out so that the user can observe what happened. + return nil, err + } + // Successfully migrated, now try to hold a read-lock on format.json + continue + } + + return rlk, nil + } +} diff --git a/cmd/format-fs_test.go b/cmd/format-fs_test.go new file mode 100644 index 000000000..213deb2e5 --- /dev/null +++ b/cmd/format-fs_test.go @@ -0,0 +1,102 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "os" + "path/filepath" + "testing" +) + +// TestFSFormatFS - tests initFormatFS, formatMetaGetFormatBackendFS, formatFSGetVersion. +func TestFSFormatFS(t *testing.T) { + // Prepare for testing + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + defer os.RemoveAll(disk) + + fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile) + + // Assign a new UUID. + uuid := mustGetUUID() + + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolumeFS(disk, uuid); err != nil { + t.Fatal(err) + } + + rlk, err := initFormatFS(disk) + if err != nil { + t.Fatal(err) + } + rlk.Close() + + // Do the basic sanity checks to check if initFormatFS() did its job. + f, err := os.OpenFile(fsFormatPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + format, err := formatMetaGetFormatBackendFS(f) + if err != nil { + t.Fatal(err) + } + if format != formatBackendFS { + t.Fatalf(`expected: %s, got: %s`, formatBackendFS, format) + } + version, err := formatFSGetVersion(f) + if err != nil { + t.Fatal(err) + } + if version != formatFSVersionV1 { + t.Fatalf(`expected: %s, got: %s`, formatFSVersionV1, version) + } + + // Corrupt the format.json file and test the functions. + // formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors. + if err = f.Truncate(0); err != nil { + t.Fatal(err) + } + if _, err = f.WriteString("b"); err != nil { + t.Fatal(err) + } + + if _, err = formatMetaGetFormatBackendFS(f); err == nil { + t.Fatal("expected to fail") + } + if _, err = formatFSGetVersion(rlk); err == nil { + t.Fatal("expected to fail") + } + if _, err = initFormatFS(disk); err == nil { + t.Fatal("expected to fail") + } + + // With unknown formatMetaV1.Version formatMetaGetFormatBackendFS, initFormatFS should return error. + if err = f.Truncate(0); err != nil { + t.Fatal(err) + } + // Here we set formatMetaV1.Version to "2" + if _, err = f.WriteString(`{"version":"2","format":"fs","fs":{"version":"1"}}`); err != nil { + t.Fatal(err) + } + if _, err = formatMetaGetFormatBackendFS(f); err == nil { + t.Fatal("expected to fail") + } + if _, err = initFormatFS(disk); err == nil { + t.Fatal("expected to fail") + } +} diff --git a/cmd/format-meta.go b/cmd/format-meta.go new file mode 100644 index 000000000..9946bed52 --- /dev/null +++ b/cmd/format-meta.go @@ -0,0 +1,52 @@ +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +// Format related consts +const ( + // Format config file carries backend format specific details. + formatConfigFile = "format.json" + + // Format config tmp file carries backend format. + formatConfigFileTmp = "format.json.tmp" +) + +const ( + // Version of the formatMetaV1 + formatMetaVersionV1 = "1" +) + +// format.json currently has the format: +// { +// "version": "1", +// "format": "XXXXX", +// "XXXXX": { +// +// } +// } +// Here "XXXXX" depends on the backend, currently we have "fs" and "xl" implementations. +// formatMetaV1 should be inherited by backend format structs. Please look at format-fs.go +// and format-xl.go for details. + +// Ideally we will never have a situation where we will have to change the +// fields of this struct and deal with related migration. +type formatMetaV1 struct { + // Version of the format config. + Version string `json:"version"` + // Format indicates the backend format type, supports two values 'xl' and 'fs'. + Format string `json:"format"` +} diff --git a/cmd/format-config-v1.go b/cmd/format-xl.go similarity index 68% rename from cmd/format-config-v1.go rename to cmd/format-xl.go index 143d6567e..4cbc7e91b 100644 --- a/cmd/format-config-v1.go +++ b/cmd/format-xl.go @@ -20,168 +20,30 @@ import ( "encoding/json" "errors" "fmt" - "io" - "io/ioutil" "reflect" "sync" - - errors2 "github.com/minio/minio/pkg/errors" - "github.com/minio/minio/pkg/lock" ) -// fsFormat - structure holding 'fs' format. -type fsFormat struct { - Version string `json:"version"` -} - -// FS format version strings. const ( - // Represents the current backend disk structure - // version under `.minio.sys` and actual data namespace. - - // formatConfigV1.fsFormat.Version - fsFormatBackendV1 = "1" -) - -// xlFormat - structure holding 'xl' format. -type xlFormat struct { - Version string `json:"version"` // Version of 'xl' format. - Disk string `json:"disk"` // Disk field carries assigned disk uuid. - // JBOD field carries the input disk order generated the first - // time when fresh disks were supplied. - JBOD []string `json:"jbod"` -} - -// XL format version strings. -const ( - // Represents the current backend disk structure - // version under `.minio.sys` and actual data namespace. - - // formatConfigV1.xlFormat.Version - xlFormatBackendV1 = "1" -) - -// formatConfigV1 - structure holds format config version '1'. -type formatConfigV1 struct { - Version string `json:"version"` // Version of the format config. - // Format indicates the backend format type, supports two values 'xl' and 'fs'. - Format string `json:"format"` - FS *fsFormat `json:"fs,omitempty"` // FS field holds fs format. - XL *xlFormat `json:"xl,omitempty"` // XL field holds xl format. -} - -// Format json file. -const ( - // Format config file carries backend format specific details. - formatConfigFile = "format.json" - - // Format config tmp file carries backend format. - formatConfigFileTmp = "format.json.tmp" -) - -// `format.json` version value. -const ( - // formatConfigV1.Version represents the version string - // of the current structure and its fields in `format.json`. - formatFileV1 = "1" - - // Future `format.json` structure changes should have - // its own version and should be subsequently listed here. -) - -// Constitutes `format.json` backend name. -const ( - // Represents FS backend. - formatBackendFS = "fs" - // Represents XL backend. formatBackendXL = "xl" + + // formatXLV1.XL.Version + formatXLVersionV1 = "1" ) -// CheckFS if the format is FS and is valid with right values -// returns appropriate errors otherwise. -func (f *formatConfigV1) CheckFS() error { - // Validate if format config version is v1. - if f.Version != formatFileV1 { - return fmt.Errorf("Unknown format file version '%s'", f.Version) - } - - // Validate if we have the expected format. - if f.Format != formatBackendFS { - return fmt.Errorf("FS backend format required. Found '%s'", f.Format) - } - - // Check if format is currently supported. - if f.FS.Version != fsFormatBackendV1 { - return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version) - } - - // Success. - return nil -} - -// LoadFormat - loads format config v1, returns `errUnformattedDisk` -// if reading format.json fails with io.EOF. -func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error { - _, err := f.ReadFrom(lk) - if errors2.Cause(err) == io.EOF { - // No data on disk `format.json` still empty - // treat it as unformatted disk. - return errors2.Trace(errUnformattedDisk) - } - return err -} - -func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) { - // Serialize to prepare to write to disk. - var fbytes []byte - fbytes, err = json.Marshal(f) - if err != nil { - return 0, errors2.Trace(err) - } - if err = lk.Truncate(0); err != nil { - return 0, errors2.Trace(err) - } - _, err = lk.Write(fbytes) - if err != nil { - return 0, errors2.Trace(err) - } - return int64(len(fbytes)), nil -} - -func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) { - var fbytes []byte - fi, err := lk.Stat() - if err != nil { - return 0, errors2.Trace(err) - } - fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size())) - if err != nil { - return 0, errors2.Trace(err) - } - if len(fbytes) == 0 { - return 0, errors2.Trace(io.EOF) - } - // Decode `format.json`. - if err = json.Unmarshal(fbytes, f); err != nil { - return 0, errors2.Trace(err) - } - return int64(len(fbytes)), nil -} - -func newFSFormat() (format *formatConfigV1) { - return newFSFormatV1() -} - -// newFSFormatV1 - initializes new formatConfigV1 with FS format info. -func newFSFormatV1() (format *formatConfigV1) { - return &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendFS, - FS: &fsFormat{ - Version: fsFormatBackendV1, - }, - } +// Represents the current backend disk structure +// version under `.minio.sys` and actual data namespace. +// formatXLV1 - structure holds format config version '1'. +type formatXLV1 struct { + formatMetaV1 + XL struct { + Version string `json:"version"` // Version of 'xl' format. + Disk string `json:"disk"` // Disk field carries assigned disk uuid. + // JBOD field carries the input disk order generated the first + // time when fresh disks were supplied. + JBOD []string `json:"jbod"` + } `json:"xl"` // XL field holds xl format. } /* @@ -271,7 +133,7 @@ func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount, } // loadAllFormats - load all format config from all input disks in parallel. -func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { +func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) { // Initialize sync waitgroup. var wg = &sync.WaitGroup{} @@ -279,7 +141,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { var sErrs = make([]error, len(bootstrapDisks)) // Initialize format configs. - var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks)) + var formats = make([]*formatXLV1, len(bootstrapDisks)) // Make a volume entry on all underlying storage disks. for index, disk := range bootstrapDisks { @@ -291,12 +153,12 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { // Make a volume inside a go-routine. go func(index int, disk StorageAPI) { defer wg.Done() - formatConfig, lErr := loadFormat(disk) + format, lErr := loadFormat(disk) if lErr != nil { sErrs[index] = lErr return } - formatConfigs[index] = formatConfig + formats[index] = format }(index, disk) } @@ -306,11 +168,11 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { for _, err := range sErrs { if err != nil { // Return all formats and errors. - return formatConfigs, sErrs + return formats, sErrs } } // Return all formats and nil - return formatConfigs, sErrs + return formats, sErrs } // genericFormatCheckXL - validates and returns error. @@ -318,7 +180,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) { // if (any disk is corrupt) return error // phase2 // if (jbod inconsistent) return error // phase2 // if (disks not recognized) // Always error. -func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err error) { +func genericFormatCheckXL(formats []*formatXLV1, sErrs []error) (err error) { // Calculate the errors. var ( errCorruptFormatCount = 0 @@ -342,20 +204,20 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e } // Calculate read quorum. - readQuorum := len(formatConfigs) / 2 + readQuorum := len(formats) / 2 // Validate the err count under read quorum. - if errCount > len(formatConfigs)-readQuorum { + if errCount > len(formats)-readQuorum { return errXLReadQuorum } // Check if number of corrupted format under read quorum - if errCorruptFormatCount > len(formatConfigs)-readQuorum { + if errCorruptFormatCount > len(formats)-readQuorum { return errCorruptedFormat } // Validates if format and JBOD are consistent across all disks. - if err = checkFormatXL(formatConfigs); err != nil { + if err = checkFormatXL(formats); err != nil { return err } @@ -366,15 +228,15 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e // isSavedUUIDInOrder - validates if disk uuid is present and valid in all // available format config JBOD. This function also validates if the disk UUID // is always available on all JBOD under the same order. -func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool { +func isSavedUUIDInOrder(uuid string, formats []*formatXLV1) bool { var orderIndexes []int // Validate each for format.json for relevant uuid. - for _, formatConfig := range formatConfigs { - if formatConfig == nil { + for _, format := range formats { + if format == nil { continue } // Validate if UUID is present in JBOD. - uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD) + uuidIndex := findDiskIndex(uuid, format.XL.JBOD) if uuidIndex == -1 { // UUID not found. errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid) @@ -398,15 +260,15 @@ func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool { } // checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks. -func checkDisksConsistency(formatConfigs []*formatConfigV1) error { - var disks = make([]string, len(formatConfigs)) +func checkDisksConsistency(formats []*formatXLV1) error { + var disks = make([]string, len(formats)) // Collect currently available disk uuids. - for index, formatConfig := range formatConfigs { - if formatConfig == nil { + for index, format := range formats { + if format == nil { disks[index] = "" continue } - disks[index] = formatConfig.XL.Disk + disks[index] = format.XL.Disk } // Validate collected uuids and verify JBOD. for _, uuid := range disks { @@ -414,7 +276,7 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error { continue } // Is uuid present on all JBOD ?. - if !isSavedUUIDInOrder(uuid, formatConfigs) { + if !isSavedUUIDInOrder(uuid, formats) { return fmt.Errorf("%s disk not found in JBOD", uuid) } } @@ -422,17 +284,17 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error { } // checkJBODConsistency - validate xl jbod order if they are consistent. -func checkJBODConsistency(formatConfigs []*formatConfigV1) error { +func checkJBODConsistency(formats []*formatXLV1) error { var sentinelJBOD []string // Extract first valid JBOD. - for _, format := range formatConfigs { + for _, format := range formats { if format == nil { continue } sentinelJBOD = format.XL.JBOD break } - for _, format := range formatConfigs { + for _, format := range formats { if format == nil { continue } @@ -458,14 +320,14 @@ func findDiskIndex(disk string, jbod []string) int { // format-config. If assignUUIDs is true, it assigns UUIDs to disks // with missing format configurations in the reference configuration. func reorderDisks(bootstrapDisks []StorageAPI, - formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1, + formats []*formatXLV1, assignUUIDs bool) (*formatXLV1, []StorageAPI, error) { // Pick first non-nil format-cfg as reference - var refCfg *formatConfigV1 - for _, formatConfig := range formatConfigs { - if formatConfig != nil { - refCfg = formatConfig + var refCfg *formatXLV1 + for _, format := range formats { + if format != nil { + refCfg = format break } } @@ -476,7 +338,7 @@ func reorderDisks(bootstrapDisks []StorageAPI, // construct reordered disk slice var newDisks = make([]StorageAPI, len(bootstrapDisks)) - for fIndex, format := range formatConfigs { + for fIndex, format := range formats { if format == nil { continue } @@ -501,7 +363,7 @@ func reorderDisks(bootstrapDisks []StorageAPI, } // loadFormat - loads format.json from disk. -func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) { +func loadFormat(disk StorageAPI) (format *formatXLV1, err error) { buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile) if err != nil { // 'file not found' and 'volume not found' as @@ -523,7 +385,7 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) { } // Try to decode format json into formatConfigV1 struct. - format = &formatConfigV1{} + format = &formatXLV1{} if err = json.Unmarshal(buf, format); err != nil { return nil, err } @@ -535,23 +397,20 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) { // collectNSaveNewFormatConfigs - creates new format configs based on // the reference config and saves it on all disks, this is to be // called from healFormatXL* functions. -func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1, +func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1, orderedDisks []StorageAPI) error { // Collect new format configs that need to be written. - var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks)) + var newFormatConfigs = make([]*formatXLV1, len(orderedDisks)) for index := range orderedDisks { // New configs are generated since we are going // to re-populate across all disks. - config := &formatConfigV1{ - Version: referenceConfig.Version, - Format: referenceConfig.Format, - XL: &xlFormat{ - Version: referenceConfig.XL.Version, - Disk: referenceConfig.XL.JBOD[index], - JBOD: referenceConfig.XL.JBOD, - }, - } + config := &formatXLV1{} + config.Version = referenceConfig.Version + config.Format = referenceConfig.Format + config.XL.Version = referenceConfig.XL.Version + config.XL.Disk = referenceConfig.XL.JBOD[index] + config.XL.JBOD = referenceConfig.XL.JBOD newFormatConfigs[index] = config } @@ -568,11 +427,11 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1, // unexpected errors as regular errors can be ignored since there // might be enough quorum to be operational. Heals only fresh disks. func healFormatXLFreshDisks(storageDisks []StorageAPI, - formatConfigs []*formatConfigV1) error { + formats []*formatXLV1) error { // Reorder the disks based on the JBOD order. referenceConfig, orderedDisks, err := reorderDisks(storageDisks, - formatConfigs, true) + formats, true) if err != nil { return err } @@ -581,7 +440,7 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI, // We need to make sure we have kept the previous order // and allowed fresh disks to be arranged anywhere. // Following block facilitates to put fresh disks. - for index, format := range formatConfigs { + for index, format := range formats { // Format is missing so we go through ordered disks. if format == nil { // At this point when disk is missing the fresh disk @@ -624,9 +483,9 @@ func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) ( // Inspect the content of all disks to guess the right order according // to the format files. The right order is represented in orderedDisks func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, - formatConfigs []*formatConfigV1) ([]StorageAPI, error) { + formats []*formatXLV1) ([]StorageAPI, error) { - for index, format := range formatConfigs { + for index, format := range formats { if format != nil { continue } @@ -681,11 +540,11 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, // Heals corrupted format json in all disks func healFormatXLCorruptedDisks(storageDisks []StorageAPI, - formatConfigs []*formatConfigV1) error { + formats []*formatXLV1) error { // Reorder the disks based on the JBOD order. referenceConfig, orderedDisks, err := reorderDisks(storageDisks, - formatConfigs, true) + formats, true) if err != nil { return err } @@ -693,7 +552,7 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI, // For disks with corrupted formats, inspect the disks // contents to guess the disks order orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, - formatConfigs) + formats) if err != nil { return err } @@ -721,7 +580,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA var unformattedDisksFoundCnt = 0 var diskNotFoundCount = 0 var corruptedDisksFoundCnt = 0 - formatConfigs := make([]*formatConfigV1, len(bootstrapDisks)) + formats := make([]*formatXLV1, len(bootstrapDisks)) // Try to load `format.json` bootstrap disks. for index, disk := range bootstrapDisks { @@ -729,7 +588,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA diskNotFoundCount++ continue } - var formatXL *formatConfigV1 + var formatXL *formatXLV1 formatXL, err = loadFormat(disk) if err != nil { if err == errUnformattedDisk { @@ -745,7 +604,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA return nil, err } // Save valid formats. - formatConfigs[index] = formatXL + formats[index] = formatXL } // If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'. @@ -760,59 +619,59 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA } // Validate the format configs read are correct. - if err = checkFormatXL(formatConfigs); err != nil { + if err = checkFormatXL(formats); err != nil { return nil, err } // Erasure code requires disks to be presented in the same // order each time. - _, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs, + _, orderedDisks, err := reorderDisks(bootstrapDisks, formats, false) return orderedDisks, err } -func checkFormatXLValue(formatXL *formatConfigV1) error { +func checkFormatXLValue(formatXL *formatXLV1) error { // Validate format version and format type. - if formatXL.Version != formatFileV1 { + if formatXL.Version != formatMetaVersionV1 { return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version) } if formatXL.Format != formatBackendXL { return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format) } - if formatXL.XL.Version != "1" { + if formatXL.XL.Version != formatXLVersionV1 { return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version) } return nil } -func checkFormatXLValues(formatConfigs []*formatConfigV1) (int, error) { - for i, formatXL := range formatConfigs { +func checkFormatXLValues(formats []*formatXLV1) (int, error) { + for i, formatXL := range formats { if formatXL == nil { continue } if err := checkFormatXLValue(formatXL); err != nil { return i, err } - if len(formatConfigs) != len(formatXL.XL.JBOD) { + if len(formats) != len(formatXL.XL.JBOD) { return i, fmt.Errorf("Number of disks %d did not match the backend format %d", - len(formatConfigs), len(formatXL.XL.JBOD)) + len(formats), len(formatXL.XL.JBOD)) } } return -1, nil } // checkFormatXL - verifies if format.json format is intact. -func checkFormatXL(formatConfigs []*formatConfigV1) error { - if _, err := checkFormatXLValues(formatConfigs); err != nil { +func checkFormatXL(formats []*formatXLV1) error { + if _, err := checkFormatXLValues(formats); err != nil { return err } - if err := checkJBODConsistency(formatConfigs); err != nil { + if err := checkJBODConsistency(formats); err != nil { return err } - return checkDisksConsistency(formatConfigs) + return checkDisksConsistency(formats) } // saveFormatXL - populates `format.json` on disks in its order. -func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error { +func saveFormatXL(storageDisks []StorageAPI, formats []*formatXLV1) error { var errs = make([]error, len(storageDisks)) var wg = &sync.WaitGroup{} // Write `format.json` to all disks. @@ -821,7 +680,7 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error { continue } wg.Add(1) - go func(index int, disk StorageAPI, format *formatConfigV1) { + go func(index int, disk StorageAPI, format *formatXLV1) { defer wg.Done() // Marshal and write to disk. @@ -862,45 +721,37 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error { return nil } -// initFormatXL - save XL format configuration on all disks. -func initFormatXL(storageDisks []StorageAPI) (err error) { - // Initialize jbods. - var jbod = make([]string, len(storageDisks)) +// Return a slice of format, to be used to format uninitialized disks. +func newFormatXLV1(diskCount int) []*formatXLV1 { + var jbod = make([]string, diskCount) - // Initialize formats. - var formats = make([]*formatConfigV1, len(storageDisks)) + var formats = make([]*formatXLV1, diskCount) - // Initialize `format.json`. - for index, disk := range storageDisks { - if disk == nil { - continue - } - // Allocate format config. - formats[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: mustGetUUID(), - }, - } - jbod[index] = formats[index].XL.Disk + for i := 0; i < diskCount; i++ { + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = mustGetUUID() + formats[i] = format + jbod[i] = formats[i].XL.Disk } // Update the jbod entries. - for index, disk := range storageDisks { - if disk == nil { - continue - } - // Save jbod. - formats[index].XL.JBOD = jbod + for i := 0; i < diskCount; i++ { + formats[i].XL.JBOD = jbod } + return formats +} + +// initFormatXL - save XL format configuration on all disks. +func initFormatXL(storageDisks []StorageAPI) (err error) { // Initialize meta volume, if volume already exists ignores it. if err := initMetaVolume(storageDisks); err != nil { return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) } // Save formats `format.json` across all disks. - return saveFormatXL(storageDisks, formats) + return saveFormatXL(storageDisks, newFormatXLV1(len(storageDisks))) } diff --git a/cmd/format-config-v1_test.go b/cmd/format-xl_test.go similarity index 74% rename from cmd/format-config-v1_test.go rename to cmd/format-xl_test.go index cedebb786..17271aa4d 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-xl_test.go @@ -18,54 +18,46 @@ package cmd import ( "bytes" - "errors" "os" - "path/filepath" "testing" - errors2 "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/hash" - "github.com/minio/minio/pkg/lock" ) // generates a valid format.json for XL backend. -func genFormatXLValid() []*formatConfigV1 { +func genFormatXLValid() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } return formatConfigs } // generates a invalid format.json version for XL backend. -func genFormatXLInvalidVersion() []*formatConfigV1 { +func genFormatXLInvalidVersion() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } // Corrupt version numbers. formatConfigs[0].Version = "2" @@ -74,46 +66,42 @@ func genFormatXLInvalidVersion() []*formatConfigV1 { } // generates a invalid format.json version for XL backend. -func genFormatXLInvalidFormat() []*formatConfigV1 { +func genFormatXLInvalidFormat() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } - // Corrupt version numbers. + // Corrupt format. formatConfigs[0].Format = "lx" formatConfigs[3].Format = "lx" return formatConfigs } // generates a invalid format.json version for XL backend. -func genFormatXLInvalidXLVersion() []*formatConfigV1 { +func genFormatXLInvalidXLVersion() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } // Corrupt version numbers. formatConfigs[0].XL.Version = "10" @@ -121,51 +109,47 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 { return formatConfigs } -func genFormatFS() *formatConfigV1 { - return &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendFS, - } +func genFormatFS() *formatXLV1 { + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendFS + return format } // generates a invalid format.json version for XL backend. -func genFormatXLInvalidJBODCount() []*formatConfigV1 { +func genFormatXLInvalidJBODCount() []*formatXLV1 { jbod := make([]string, 7) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } return formatConfigs } // generates a invalid format.json JBOD for XL backend. -func genFormatXLInvalidJBOD() []*formatConfigV1 { +func genFormatXLInvalidJBOD() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } for index := range jbod { jbod[index] = mustGetUUID() @@ -177,22 +161,20 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 { } // generates a invalid format.json Disk UUID for XL backend. -func genFormatXLInvalidDisks() []*formatConfigV1 { +func genFormatXLInvalidDisks() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } // Make disk 5 and disk 8 have inconsistent disk uuid's. formatConfigs[4].XL.Disk = mustGetUUID() @@ -201,22 +183,20 @@ func genFormatXLInvalidDisks() []*formatConfigV1 { } // generates a invalid format.json Disk UUID in wrong order for XL backend. -func genFormatXLInvalidDisksOrder() []*formatConfigV1 { +func genFormatXLInvalidDisksOrder() []*formatXLV1 { jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } // Re order jbod for failure case. var jbod1 = make([]string, 8) @@ -459,7 +439,7 @@ func TestFormatXLReorderByInspection(t *testing.T) { // - invalid JBOD // - invalid Disk uuid func TestFormatXL(t *testing.T) { - formatInputCases := [][]*formatConfigV1{ + formatInputCases := [][]*formatXLV1{ genFormatXLValid(), genFormatXLInvalidVersion(), genFormatXLInvalidFormat(), @@ -470,7 +450,7 @@ func TestFormatXL(t *testing.T) { genFormatXLInvalidDisksOrder(), } testCases := []struct { - formatConfigs []*formatConfigV1 + formatConfigs []*formatXLV1 shouldPass bool }{ { @@ -525,22 +505,20 @@ func TestSavedUUIDOrder(t *testing.T) { shouldPass bool }, 8) jbod := make([]string, 8) - formatConfigs := make([]*formatConfigV1, 8) + formatConfigs := make([]*formatXLV1, 8) for index := range jbod { jbod[index] = mustGetUUID() uuidTestCases[index].uuid = jbod[index] uuidTestCases[index].shouldPass = true } for index := range jbod { - formatConfigs[index] = &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendXL, - XL: &xlFormat{ - Version: xlFormatBackendV1, - Disk: jbod[index], - JBOD: jbod, - }, - } + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendXL + format.XL.Version = formatXLVersionV1 + format.XL.Disk = jbod[index] + format.XL.JBOD = jbod + formatConfigs[index] = format } // Re order jbod for failure case. var jbod1 = make([]string, 8) @@ -653,172 +631,18 @@ func TestGenericFormatCheckXL(t *testing.T) { t.Fatalf("Should fail here") } errs = []error{nil} - if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil { + format := &formatXLV1{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendFS + if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil { t.Fatalf("Should fail here") } errs = []error{errFaultyDisk} - if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil { + if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil { t.Fatalf("Should fail here") } } -// TestFSCheckFormatFSErr - test loadFormatFS loading older format. -func TestFSCheckFormatFSErr(t *testing.T) { - // Prepare for testing - disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) - defer os.RemoveAll(disk) - - // Assign a new UUID. - uuid := mustGetUUID() - - // Initialize meta volume, if volume already exists ignores it. - if err := initMetaVolumeFS(disk, uuid); err != nil { - t.Fatal(err) - } - - testCases := []struct { - format *formatConfigV1 - formatWriteErr error - formatCheckErr error - shouldPass bool - }{ - { - format: &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendFS, - FS: &fsFormat{ - Version: fsFormatBackendV1, - }, - }, - formatCheckErr: nil, - shouldPass: true, - }, - { - format: &formatConfigV1{ - Version: formatFileV1, - Format: formatBackendFS, - FS: &fsFormat{ - Version: "10", - }, - }, - formatCheckErr: errors.New("Unknown backend FS format version '10'"), - shouldPass: false, - }, - { - format: &formatConfigV1{ - Version: formatFileV1, - Format: "garbage", - FS: &fsFormat{ - Version: fsFormatBackendV1, - }, - }, - formatCheckErr: errors.New("FS backend format required. Found 'garbage'"), - }, - { - format: &formatConfigV1{ - Version: "-1", - Format: formatBackendFS, - FS: &fsFormat{ - Version: fsFormatBackendV1, - }, - }, - formatCheckErr: errors.New("Unknown format file version '-1'"), - }, - } - - fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile) - for i, testCase := range testCases { - lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - t.Fatal(err) - } - _, err = testCase.format.WriteTo(lk) - lk.Close() - if err != nil { - t.Fatalf("Test %d: Expected nil, got %s", i+1, err) - } - - lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - t.Fatal(err) - } - - formatCfg := &formatConfigV1{} - _, err = formatCfg.ReadFrom(lk) - lk.Close() - if err != nil { - t.Fatal(err) - } - err = formatCfg.CheckFS() - if err != nil && testCase.shouldPass { - t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err) - } - if err == nil && !testCase.shouldPass { - t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr) - } - if err != nil && !testCase.shouldPass { - if errors2.Cause(err).Error() != testCase.formatCheckErr.Error() { - t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err) - } - } - } -} - -// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks -func TestFSCheckFormatFS(t *testing.T) { - // Prepare for testing - disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) - defer os.RemoveAll(disk) - - // Assign a new UUID. - uuid := mustGetUUID() - - // Initialize meta volume, if volume already exists ignores it. - if err := initMetaVolumeFS(disk, uuid); err != nil { - t.Fatal(err) - } - - fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile) - lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - t.Fatal(err) - } - - format := newFSFormatV1() - _, err = format.WriteTo(lk) - lk.Close() - if err != nil { - t.Fatal(err) - } - - // Loading corrupted format file - file, err := os.OpenFile((fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - t.Fatal("Should not fail here", err) - } - file.Write([]byte{'b'}) - file.Close() - - lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - t.Fatal(err) - } - - format = &formatConfigV1{} - _, err = format.ReadFrom(lk) - lk.Close() - if err == nil { - t.Fatal("Should return an error here") - } - - // Loading format file from disk not found. - os.RemoveAll(disk) - _, err = lock.LockedOpenFile((fsFormatPath), os.O_RDONLY, 0600) - if err != nil && !os.IsNotExist(err) { - t.Fatal("Should return 'format.json' does not exist, but got", err) - } -} - func TestLoadFormatXLErrs(t *testing.T) { nDisks := 16 fsDirs, err := getRandomDisks(nDisks) diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index c2b1b8874..b90b6208f 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -24,7 +24,6 @@ import ( pathutil "path" "sort" "strings" - "time" "github.com/minio/minio/pkg/errors" "github.com/minio/minio/pkg/lock" @@ -269,97 +268,6 @@ func newFSMetaV1() (fsMeta fsMetaV1) { return fsMeta } -// Check if disk has already a valid format, holds a read lock and -// upon success returns it to the caller to be closed. -func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) { - fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile) - - rlk, err := lock.RLockedOpenFile((fsFormatPath)) - if err != nil { - if os.IsNotExist(err) { - // If format.json not found then - // its an unformatted disk. - return nil, errors.Trace(errUnformattedDisk) - } - return nil, errors.Trace(err) - } - - var format = &formatConfigV1{} - if err = format.LoadFormat(rlk.LockedFile); err != nil { - rlk.Close() - return nil, err - } - - // Check format FS. - if err = format.CheckFS(); err != nil { - rlk.Close() - return nil, err - } - - // Always return read lock here and should be closed by the caller. - return rlk, errors.Trace(err) -} - -// Creates a new format.json if unformatted. -func createFormatFS(fsPath string) error { - fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile) - - // Attempt a write lock on formatConfigFile `format.json` - // file stored in minioMetaBucket(.minio.sys) directory. - lk, err := lock.TryLockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return errors.Trace(err) - } - // Close the locked file upon return. - defer lk.Close() - - // Load format on disk, checks if we are unformatted - // writes the new format.json - var format = &formatConfigV1{} - err = format.LoadFormat(lk) - if errors.Cause(err) == errUnformattedDisk { - _, err = newFSFormat().WriteTo(lk) - return err - } - return err -} - -func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) { - // This loop validates format.json by holding a read lock and - // proceeds if disk unformatted to hold non-blocking WriteLock - // If for some reason non-blocking WriteLock fails and the error - // is lock.ErrAlreadyLocked i.e some other process is holding a - // lock we retry in the loop again. - for { - // Validate the `format.json` for expected values. - rlk, err = checkLockedValidFormatFS(fsPath) - switch { - case err == nil: - // Holding a read lock ensures that any write lock operation - // is blocked if attempted in-turn avoiding corruption on - // the backend disk. - return rlk, nil - case errors.Cause(err) == errUnformattedDisk: - if err = createFormatFS(fsPath); err != nil { - // Existing write locks detected. - if errors.Cause(err) == lock.ErrAlreadyLocked { - // Lock already present, sleep and attempt again. - time.Sleep(100 * time.Millisecond) - continue - } - - // Unexpected error, return. - return nil, err - } - - // Loop will continue to attempt a read-lock on `format.json`. - default: - // Unhandled error return. - return nil, err - } - } -} - // Return if the part info in uploadedParts and CompleteParts are same. func isPartsSame(uploadedParts []objectPartInfo, CompleteParts []CompletePart) bool { if len(uploadedParts) != len(CompleteParts) { diff --git a/cmd/utils.go b/cmd/utils.go index 74b05f642..ca5ef5436 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -263,3 +263,11 @@ func NewCustomHTTPTransport() http.RoundTripper { DisableCompression: true, } } + +// Load the json (typically from disk file). +func jsonLoadFromSeeker(r io.ReadSeeker, data interface{}) error { + if _, err := r.Seek(0, io.SeekStart); err != nil { + return err + } + return json.NewDecoder(r).Decode(data) +} diff --git a/cmd/utils_test.go b/cmd/utils_test.go index db7dbd6e7..41fa1a763 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -17,6 +17,7 @@ package cmd import ( + "bytes" "encoding/json" "errors" "io/ioutil" @@ -363,3 +364,19 @@ func TestContains(t *testing.T) { } } } + +// Test jsonLoadFromSeeker. +func TestJSONLoadFromSeeker(t *testing.T) { + format := newFormatFSV1() + b, err := json.Marshal(format) + if err != nil { + t.Fatal(err) + } + var gotFormat formatFSV1 + if err = jsonLoadFromSeeker(bytes.NewReader(b), &gotFormat); err != nil { + t.Fatal(err) + } + if *format != gotFormat { + t.Fatal("jsonLoadFromSeeker() failed to decode json") + } +}