MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-07 04:42:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: drive sub-sys to max timeout reload (#18501)

Browse Source
This commit is contained in:
jiuker
2023-11-28 01:15:06 +08:00
committed by GitHub
parent 506f121576
commit be02333529
12 changed files with 197 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/bitrot-streaming.go
View File

@@ -95,7 +95,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
h := algo.New()
bw := &streamingBitrotWriter{
iow: ioutil.NewDeadlineWriter(w, diskMaxTimeout),
iow: ioutil.NewDeadlineWriter(w, globalDriveConfig.GetMaxTimeout()),
closeWithErr: w.CloseWithError,
h: h,
shardSize: shardSize,

20
cmd/config-current.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/minio/minio/internal/config/callhome"
"github.com/minio/minio/internal/config/compress"
"github.com/minio/minio/internal/config/dns"
"github.com/minio/minio/internal/config/drive"
"github.com/minio/minio/internal/config/etcd"
"github.com/minio/minio/internal/config/heal"
xldap "github.com/minio/minio/internal/config/identity/ldap"
@@ -69,6 +70,7 @@ func initHelp() {
config.ScannerSubSys: scanner.DefaultKVS,
config.SubnetSubSys: subnet.DefaultKVS,
config.CallhomeSubSys: callhome.DefaultKVS,
config.DriveSubSys: drive.DefaultKVS,
config.CacheSubSys: cache.DefaultKVS,
}
for k, v := range notify.DefaultNotificationKVS {
@@ -97,6 +99,10 @@ func initHelp() {
Description: "enable callhome to MinIO SUBNET",
Optional: true,
},
config.HelpKV{
Key: config.DriveSubSys,
Description: "enable drive specific settings",
},
config.HelpKV{
Key: config.SiteSubSys,
Description: "label the server and its location",
@@ -258,6 +264,7 @@ func initHelp() {
config.LambdaWebhookSubSys: lambda.HelpWebhook,
config.SubnetSubSys: subnet.HelpSubnet,
config.CallhomeSubSys: callhome.HelpCallhome,
config.DriveSubSys: drive.HelpDrive,
config.CacheSubSys: cache.Help,
}
@@ -366,6 +373,10 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
if cfg.Enabled() && !globalSubnetConfig.Registered() {
return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
}
case config.DriveSubSys:
if _, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil {
return err
}
case config.CacheSubSys:
if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport); err != nil {
return err
@@ -645,6 +656,15 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
initCallhome(ctx, objAPI)
}
}
case config.DriveSubSys:
if driveConfig, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err))
} else {
err := globalDriveConfig.Update(driveConfig)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
}
}
case config.CacheSubSys:
cacheCfg, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport)
if err != nil {

6
cmd/erasure-multipart.go
View File

@@ -204,7 +204,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
if err != nil {
return nil
}
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
wait := deletedCleanupSleeper.Timer(ctx)
if now.Sub(fi.ModTime) > expiry {
@@ -218,7 +218,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
if err != nil {
return nil
}
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
wait := deletedCleanupSleeper.Timer(ctx)
if now.Sub(vi.Created) > expiry {
@@ -239,7 +239,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
if err != nil {
return nil
}
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
wait := deletedCleanupSleeper.Timer(ctx)
if now.Sub(vi.Created) > expiry {

2
cmd/erasure.go
View File

@@ -342,7 +342,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
defer wg.Done()
diskPath := disk.Endpoint().Path
readDirFn(pathJoin(diskPath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
wait := deletedCleanupSleeper.Timer(ctx)
removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))

4
cmd/globals.go
View File

@@ -42,6 +42,7 @@ import (
"github.com/minio/minio/internal/config/callhome"
"github.com/minio/minio/internal/config/compress"
"github.com/minio/minio/internal/config/dns"
"github.com/minio/minio/internal/config/drive"
idplugin "github.com/minio/minio/internal/config/identity/plugin"
polplugin "github.com/minio/minio/internal/config/policy/plugin"
"github.com/minio/minio/internal/config/storageclass"
@@ -243,6 +244,9 @@ var (
// The global callhome config
globalCallhomeConfig callhome.Config
// The global drive config
globalDriveConfig drive.Config
// The global cache config
globalCacheConfig cache.Config

73
cmd/xl-storage-disk-id-check.go
View File

@@ -351,7 +351,7 @@ func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...strin
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.MakeVolBulk(ctx, volumes...) })
}
@@ -362,7 +362,7 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.MakeVol(ctx, volume) })
}
@@ -383,7 +383,7 @@ func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
err = w.Run(func() error {
var ierr error
vol, ierr = p.storage.StatVol(ctx, volume)
@@ -399,7 +399,7 @@ func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, for
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.DeleteVol(ctx, volume, forceDelete) })
}
@@ -421,7 +421,7 @@ func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
err = w.Run(func() error {
n, err = p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
return err
@@ -438,7 +438,7 @@ func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, pa
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
return p.storage.AppendFile(ctx, volume, path, buf)
})
@@ -451,7 +451,7 @@ func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path stri
}
defer done(&err)
return p.storage.CreateFile(ctx, volume, path, size, xioutil.NewDeadlineReader(io.NopCloser(reader), diskMaxTimeout))
return p.storage.CreateFile(ctx, volume, path, size, xioutil.NewDeadlineReader(io.NopCloser(reader), globalDriveConfig.GetMaxTimeout()))
}
func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
@@ -461,7 +461,7 @@ func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
var rc io.ReadCloser
err = w.Run(func() error {
@@ -473,7 +473,7 @@ func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path
return nil, err
}
return xioutil.NewDeadlineReader(rc, diskMaxTimeout), nil
return xioutil.NewDeadlineReader(rc, globalDriveConfig.GetMaxTimeout()), nil
}
func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
@@ -483,7 +483,7 @@ func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPat
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) })
}
@@ -494,7 +494,7 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
err = w.Run(func() error {
var ierr error
sign, ierr = p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
@@ -510,7 +510,7 @@ func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, pa
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.CheckParts(ctx, volume, path, fi) })
}
@@ -521,7 +521,7 @@ func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path s
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.Delete(ctx, volume, path, deleteOpts) })
}
@@ -571,7 +571,7 @@ func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.WriteAll(ctx, volume, path, b) })
}
@@ -582,7 +582,7 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker) })
}
@@ -593,7 +593,7 @@ func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) })
}
@@ -604,7 +604,7 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path s
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.WriteMetadata(ctx, volume, path, fi) })
}
@@ -615,7 +615,7 @@ func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, ve
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
rerr := w.Run(func() error {
fi, err = p.storage.ReadVersion(ctx, volume, path, versionID, opts)
return err
@@ -633,7 +633,7 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
rerr := w.Run(func() error {
buf, err = p.storage.ReadAll(ctx, volume, path)
return err
@@ -651,7 +651,7 @@ func (p *xlStorageDiskIDCheck) ReadXL(ctx context.Context, volume string, path s
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
rerr := w.Run(func() error {
rf, err = p.storage.ReadXL(ctx, volume, path, readData)
return err
@@ -696,7 +696,7 @@ func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume st
}
defer done(&err)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return p.storage.CleanAbandonedData(ctx, volume, path) })
}
@@ -769,27 +769,10 @@ const (
diskHealthFaulty
)
// diskMaxTimeoutOperation maximum wait time before we consider a drive
// offline under active monitoring.
var diskMaxTimeout = 2 * time.Minute
// diskActiveMonitoring indicates if we have enabled "active" disk monitoring
var diskActiveMonitoring = true
func init() {
d := env.Get("_MINIO_DRIVE_MAX_TIMEOUT", "")
if d == "" {
d = env.Get("_MINIO_DISK_MAX_TIMEOUT", "")
}
if d != "" {
timeoutOperation, _ := time.ParseDuration(d)
if timeoutOperation < time.Second {
logger.Info("invalid _MINIO_DISK_MAX_TIMEOUT value: %s, minimum allowed is 1s, defaulting to '2 minutes'", d)
} else {
diskMaxTimeout = timeoutOperation
}
}
diskActiveMonitoring = (env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn) ||
(env.Get("_MINIO_DISK_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn)
}
@@ -1023,19 +1006,19 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
// if disk max timeout is smaller than checkEvery window
// reduce checks by a second.
if diskMaxTimeout <= checkEvery {
checkEvery = diskMaxTimeout - time.Second
if globalDriveConfig.GetMaxTimeout() <= checkEvery {
checkEvery = globalDriveConfig.GetMaxTimeout() - time.Second
if checkEvery <= 0 {
checkEvery = diskMaxTimeout
checkEvery = globalDriveConfig.GetMaxTimeout()
}
}
// if disk max timeout is smaller than skipIfSuccessBefore window
// reduce the skipIfSuccessBefore by a second.
if diskMaxTimeout <= skipIfSuccessBefore {
skipIfSuccessBefore = diskMaxTimeout - time.Second
if globalDriveConfig.GetMaxTimeout() <= skipIfSuccessBefore {
skipIfSuccessBefore = globalDriveConfig.GetMaxTimeout() - time.Second
if skipIfSuccessBefore <= 0 {
skipIfSuccessBefore = diskMaxTimeout
skipIfSuccessBefore = globalDriveConfig.GetMaxTimeout()
}
}
@@ -1074,7 +1057,7 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
dctx, dcancel := context.WithCancel(ctx)
started := time.Now()
go func() {
timeout := time.NewTimer(diskMaxTimeout)
timeout := time.NewTimer(globalDriveConfig.GetMaxTimeout())
select {
case <-dctx.Done():
if !timeout.Stop() {

6
cmd/xl-storage.go
View File

@@ -447,7 +447,7 @@ func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string)
func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte, error) {
var buf []byte
err := xioutil.NewDeadlineWorker(diskMaxTimeout).Run(func() error {
err := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()).Run(func() error {
var rerr error
buf, _, rerr = s.readMetadataWithDMTime(ctx, itemPath)
return rerr
@@ -1085,7 +1085,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
errs[i] = ctx.Err()
continue
}
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
if err := w.Run(func() error { return s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...) }); err != nil {
errs[i] = err
}
@@ -2751,7 +2751,7 @@ func (s *xlStorage) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp
var data []byte
var mt time.Time
fullPath := pathJoin(volumeDir, req.Prefix, f)
w := xioutil.NewDeadlineWorker(diskMaxTimeout)
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
if err := w.Run(func() (err error) {
if req.MetadataOnly {
data, mt, err = s.readMetadataWithDMTime(ctx, fullPath)