fix: possible connection leaks in sets init, heal (#9263)

This commit is contained in:
Harshavardhana 2020-04-03 18:06:31 -07:00 committed by GitHub
parent c6e62b9175
commit 4714958e99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 30 additions and 12 deletions

View File

@ -221,10 +221,16 @@ func IsServerResolvable(endpoint Endpoint) error {
// connect to list of endpoints and load all XL disk formats, validate the formats are correct // connect to list of endpoints and load all XL disk formats, validate the formats are correct
// and are in quorum, if no formats are found attempt to initialize all of them for the first // and are in quorum, if no formats are found attempt to initialize all of them for the first
// time. additionally make sure to close all the disks used in this attempt. // time. additionally make sure to close all the disks used in this attempt.
func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) ([]StorageAPI, *formatXLV3, error) { func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (storageDisks []StorageAPI, format *formatXLV3, err error) {
// Initialize all storage disks // Initialize all storage disks
storageDisks, errs := initStorageDisksWithErrors(endpoints) storageDisks, errs := initStorageDisksWithErrors(endpoints)
defer func(storageDisks []StorageAPI) {
if err != nil {
closeStorageDisks(storageDisks)
}
}(storageDisks)
for i, err := range errs { for i, err := range errs {
if err != nil { if err != nil {
if err != errDiskNotFound { if err != errDiskNotFound {
@ -256,7 +262,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
// most part unless one of the formats is not consistent // most part unless one of the formats is not consistent
// with expected XL format. For example if a user is // with expected XL format. For example if a user is
// trying to pool FS backend into an XL set. // trying to pool FS backend into an XL set.
if err := checkFormatXLValues(formatConfigs, drivesPerSet); err != nil { if err = checkFormatXLValues(formatConfigs, drivesPerSet); err != nil {
return nil, nil, err return nil, nil, err
} }
@ -266,7 +272,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
zoneCount, setCount, drivesPerSet) zoneCount, setCount, drivesPerSet)
// Initialize erasure code format on disks // Initialize erasure code format on disks
format, err := initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet, deploymentID) format, err = initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet, deploymentID)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -293,7 +299,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
// This migration failed to capture '.This' field properly which indicates // This migration failed to capture '.This' field properly which indicates
// the disk UUID association. Below function is called to handle and fix // the disk UUID association. Below function is called to handle and fix
// this regression, for more info refer https://github.com/minio/minio/issues/5667 // this regression, for more info refer https://github.com/minio/minio/issues/5667
if err := fixFormatXLV3(storageDisks, endpoints, formatConfigs); err != nil { if err = fixFormatXLV3(storageDisks, endpoints, formatConfigs); err != nil {
return nil, nil, err return nil, nil, err
} }
@ -302,7 +308,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
return nil, nil, errXLV3ThisEmpty return nil, nil, errXLV3ThisEmpty
} }
format, err := getFormatXLInQuorum(formatConfigs) format, err = getFormatXLInQuorum(formatConfigs)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }

View File

@ -238,6 +238,9 @@ func (s *xlSets) connectDisks() {
} }
disk.SetDiskID(format.XL.This) disk.SetDiskID(format.XL.This)
s.xlDisksMu.Lock() s.xlDisksMu.Lock()
if s.xlDisks[setIndex][diskIndex] != nil {
s.xlDisks[setIndex][diskIndex].Close()
}
s.xlDisks[setIndex][diskIndex] = disk s.xlDisks[setIndex][diskIndex] = disk
s.xlDisksMu.Unlock() s.xlDisksMu.Unlock()
go func(setIndex int) { go func(setIndex int) {
@ -333,16 +336,21 @@ func newXLSets(endpoints Endpoints, storageDisks []StorageAPI, format *formatXLV
endpoints = append(endpoints, s.endpoints[i*drivesPerSet+j]) endpoints = append(endpoints, s.endpoints[i*drivesPerSet+j])
// Rely on endpoints list to initialize, init lockers and available disks. // Rely on endpoints list to initialize, init lockers and available disks.
s.xlLockers[i][j] = newLockAPI(s.endpoints[i*drivesPerSet+j]) s.xlLockers[i][j] = newLockAPI(s.endpoints[i*drivesPerSet+j])
if storageDisks[i*drivesPerSet+j] == nil { disk := storageDisks[i*drivesPerSet+j]
if disk == nil {
continue continue
} }
if diskID, derr := storageDisks[i*drivesPerSet+j].GetDiskID(); derr == nil { diskID, derr := disk.GetDiskID()
m, n, err := findDiskIndexByDiskID(format, diskID) if derr != nil {
if err != nil { disk.Close()
continue continue
}
s.xlDisks[m][n] = storageDisks[i*drivesPerSet+j]
} }
m, n, err := findDiskIndexByDiskID(format, diskID)
if err != nil {
disk.Close()
continue
}
s.xlDisks[m][n] = disk
} }
// Initialize xl objects for a given set. // Initialize xl objects for a given set.
@ -1375,11 +1383,13 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
diskID, err := disk.GetDiskID() diskID, err := disk.GetDiskID()
if err != nil { if err != nil {
disk.Close()
continue continue
} }
m, n, err := findDiskIndexByDiskID(refFormat, diskID) m, n, err := findDiskIndexByDiskID(refFormat, diskID)
if err != nil { if err != nil {
disk.Close()
continue continue
} }
@ -1583,11 +1593,13 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
diskID, err := disk.GetDiskID() diskID, err := disk.GetDiskID()
if err != nil { if err != nil {
disk.Close()
continue continue
} }
m, n, err := findDiskIndexByDiskID(refFormat, diskID) m, n, err := findDiskIndexByDiskID(refFormat, diskID)
if err != nil { if err != nil {
disk.Close()
continue continue
} }