xl: All nodes create meta volumes in its local disks (#8786)

Meta volumes directories, tmp/, background-ops/, etc..
undr .minio.sys are created when disks are formatted
but also when the cluster is started.

However using MakeVolBulk() is not appropriate in the
case of a user migrating from a version which does not
have .minio.sys/background-ops/. The reason is that
MakeVolBulk() exits early when an error is occured:
errVolumeExists in this case, which is expected since
some directories such as tmp/ already exist.

This commit will avoid use MakeVolBulk and use MakeVol
instead.

Also the PR will make each node creates meta volumes
in its local disks and stop relying on the first disk
since the first node could be offline.
This commit is contained in:
Anis Elleuch 2020-01-15 21:36:52 +01:00 committed by Harshavardhana
parent 442e1698cb
commit 069876e262
3 changed files with 22 additions and 25 deletions

View File

@ -621,23 +621,28 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets) return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
} }
// Initializes meta volume on all input storage disks. // Initializes meta volume only on local storage disks.
func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error { func initXLMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatXLV3) error {
// This happens for the first time, but keep this here since this
// is the only place where it can be made expensive optimizing all // Compute the local disks eligible for meta volumes (re)initialization
// other calls. Create minio meta volume, if it doesn't exist yet. var disksToInit []StorageAPI
for index := range storageDisks {
if formats[index] == nil || storageDisks[index] == nil || storageDisks[index].Hostname() != "" {
// Ignore create meta volume on disks which are not found or not local.
continue
}
disksToInit = append(disksToInit, storageDisks[index])
}
// Initialize errs to collect errors inside go-routine. // Initialize errs to collect errors inside go-routine.
g := errgroup.WithNErrs(len(storageDisks)) g := errgroup.WithNErrs(len(disksToInit))
// Initialize all disks in parallel. // Initialize all disks in parallel.
for index := range storageDisks { for index := range disksToInit {
// Initialize a new index variable in each loop so each
// goroutine will return its own instance of index variable.
index := index index := index
g.Go(func() error { g.Go(func() error {
if formats[index] == nil || storageDisks[index] == nil {
// Ignore create meta volume on disks which are not found.
return nil
}
return makeFormatXLMetaVolumes(storageDisks[index]) return makeFormatXLMetaVolumes(storageDisks[index])
}, index) }, index)
} }
@ -821,13 +826,7 @@ func ecDrivesNoConfig(drivesPerSet int) int {
// Make XL backend meta volumes. // Make XL backend meta volumes.
func makeFormatXLMetaVolumes(disk StorageAPI) error { func makeFormatXLMetaVolumes(disk StorageAPI) error {
// Attempt to create MinIO internal buckets. // Attempt to create MinIO internal buckets.
err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket) return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
if err != nil {
if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
return err
}
}
return nil
} }
var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists) var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

View File

@ -101,7 +101,7 @@ func TestFixFormatV3(t *testing.T) {
formats[j] = newFormat formats[j] = newFormat
} }
if err = initFormatXLMetaVolume(storageDisks, formats); err != nil { if err = initXLMetaVolumesInLocalDisks(storageDisks, formats); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -267,12 +267,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
return format, nil return format, nil
} }
// The first will always recreate some directories inside .minio.sys
// such as, tmp, multipart and background-ops
if firstDisk {
initFormatXLMetaVolume(storageDisks, formatConfigs)
}
// Return error when quorum unformatted disks - indicating we are // Return error when quorum unformatted disks - indicating we are
// waiting for first server to be online. // waiting for first server to be online.
if quorumUnformattedDisks(sErrs) && !firstDisk { if quorumUnformattedDisks(sErrs) && !firstDisk {
@ -331,6 +325,10 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
return nil, err return nil, err
} }
// The will always recreate some directories inside .minio.sys of
// the local disk such as tmp, multipart and background-ops
initXLMetaVolumesInLocalDisks(storageDisks, formatConfigs)
return format, nil return format, nil
} }