mirror of
https://github.com/minio/minio.git
synced 2025-11-09 21:49:46 -05:00
Make sure to close the disk connections (#5752)
Since we do not re-use storageDisks after moving the connections to object layer we should close them appropriately otherwise we have a lot of connection leaks and these can compound as the time goes by. This PR also refactors the initialization code to re-use storageDisks for given set of endpoints until we have confirmed a valid reference format.
This commit is contained in:
Harshavardhana
committed by
Nitish Tiwari
Nitish Tiwari
parent
d67e423a32
commit
85a57d2021
@@ -88,6 +88,95 @@ func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validate reference format against list of XL formats.
|
||||
func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints EndpointList, setCount, drivesPerSet int) error {
|
||||
for i := range formats {
|
||||
if formats[i] == nil {
|
||||
continue
|
||||
}
|
||||
if err := formatXLV3Check(format, formats[i]); err != nil {
|
||||
return fmt.Errorf("%s format error: %s", endpoints[i], err)
|
||||
}
|
||||
}
|
||||
if len(format.XL.Sets) != setCount {
|
||||
return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), setCount)
|
||||
}
|
||||
if len(format.XL.Sets[0]) != drivesPerSet {
|
||||
return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), drivesPerSet)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Following error message is added to fix a regression in release
|
||||
// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This
|
||||
// migration failed to capture '.This' field properly which indicates
|
||||
// the disk UUID association. Below error message is returned when
|
||||
// we see this situation in format.json, for more info refer
|
||||
// https://github.com/minio/minio/issues/5667
|
||||
var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")
|
||||
|
||||
// connect to list of endpoints and load all XL disk formats, validate the formats are correct
|
||||
// and are in quorum, if no formats are found attempt to initialize all of them for the first
|
||||
// time. additionally make sure to close all the disks used in this attempt.
|
||||
func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, drivesPerSet int) (*formatXLV3, error) {
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeStorageDisks(storageDisks)
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(storageDisks)
|
||||
|
||||
// Pre-emptively check if one of the formatted disks
|
||||
// is invalid. This function returns success for the
|
||||
// most part unless one of the formats is not consistent
|
||||
// with expected XL format. For example if a user is
|
||||
// trying to pool FS backend into an XL set.
|
||||
if err = checkFormatXLValues(formatConfigs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[errors.Cause(sErr)]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
if shouldInitXLDisks(sErrs) {
|
||||
if !firstDisk {
|
||||
return nil, errNotFirstDisk
|
||||
}
|
||||
return initFormatXL(storageDisks, setCount, drivesPerSet)
|
||||
}
|
||||
|
||||
// Following function is added to fix a regressions which was introduced
|
||||
// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
|
||||
// This migration failed to capture '.This' field properly which indicates
|
||||
// the disk UUID association. Below function is called to handle and fix
|
||||
// this regression, for more info refer https://github.com/minio/minio/issues/5667
|
||||
if err = fixFormatXLV3(storageDisks, endpoints, formatConfigs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If any of the .This field is still empty, we return error.
|
||||
if formatXLV3ThisEmpty(formatConfigs) {
|
||||
return nil, errXLV3ThisEmpty
|
||||
}
|
||||
|
||||
format, err := getFormatXLInQuorum(formatConfigs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate all format configs with reference format.
|
||||
if err = validateXLFormats(format, formatConfigs, endpoints, setCount, drivesPerSet); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return format, nil
|
||||
}
|
||||
|
||||
// Format disks before initialization of object layer.
|
||||
func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) {
|
||||
if len(endpoints) == 0 || setCount == 0 || disksPerSet == 0 {
|
||||
@@ -121,65 +210,26 @@ func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerS
|
||||
for {
|
||||
select {
|
||||
case _ = <-retryTimerCh:
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(endpoints)
|
||||
|
||||
// Pre-emptively check if one of the formatted disks
|
||||
// is invalid. This function returns success for the
|
||||
// most part unless one of the formats is not consistent
|
||||
// with expected XL format. For example if a user is
|
||||
// trying to pool FS backend into an XL set.
|
||||
if err = checkFormatXLValues(formatConfigs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[errors.Cause(sErr)]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
if shouldInitXLDisks(sErrs) {
|
||||
if !firstDisk {
|
||||
format, err := connectLoadInitFormats(firstDisk, endpoints, setCount, disksPerSet)
|
||||
if err != nil {
|
||||
switch err {
|
||||
case errNotFirstDisk:
|
||||
// Fresh setup, wait for first server to be up.
|
||||
console.Println("Waiting for the first server to format the disks.")
|
||||
continue
|
||||
case errXLReadQuorum:
|
||||
// no quorum available continue to wait for minimum number of servers.
|
||||
console.Printf("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
|
||||
continue
|
||||
case errXLV3ThisEmpty:
|
||||
// need to wait for this error to be healed, so continue.
|
||||
continue
|
||||
default:
|
||||
// For all other unhandled errors we exit and fail.
|
||||
return nil, err
|
||||
}
|
||||
return initFormatXL(endpoints, setCount, disksPerSet)
|
||||
}
|
||||
|
||||
// Following function is added to fix a regressions which was introduced
|
||||
// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
|
||||
// This migration failed to capture '.This' field properly which indicates
|
||||
// the disk UUID association. Below function is called to handle and fix
|
||||
// this regression, for more info refer https://github.com/minio/minio/issues/5667
|
||||
if err = fixFormatXLV3(endpoints, formatConfigs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If any of the .This field is still empty we wait them to be fixed.
|
||||
if formatXLV3ThisEmpty(formatConfigs) {
|
||||
continue
|
||||
}
|
||||
|
||||
format, err = getFormatXLInQuorum(formatConfigs)
|
||||
if err == nil {
|
||||
for i := range formatConfigs {
|
||||
if formatConfigs[i] == nil {
|
||||
continue
|
||||
}
|
||||
if err = formatXLV3Check(format, formatConfigs[i]); err != nil {
|
||||
return nil, fmt.Errorf("%s format error: %s", endpoints[i], err)
|
||||
}
|
||||
}
|
||||
if len(format.XL.Sets) != globalXLSetCount {
|
||||
return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), globalXLSetCount)
|
||||
}
|
||||
if len(format.XL.Sets[0]) != globalXLSetDriveCount {
|
||||
return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), globalXLSetDriveCount)
|
||||
}
|
||||
return format, nil
|
||||
}
|
||||
console.Printf("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
|
||||
return format, nil
|
||||
case <-globalOSSignalCh:
|
||||
return nil, fmt.Errorf("Initializing data volumes gracefully stopped")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user