mirror of
https://github.com/minio/minio.git
synced 2024-12-24 06:05:55 -05:00
expire lockers if lockers are offline (#10749)
lockers currently might leave stale lockers, in unknown ways waiting for downed lockers. locker check interval is high enough to safely cleanup stale locks.
This commit is contained in:
parent
6a8c62f9fd
commit
d9db7f3308
@ -348,7 +348,7 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessResponseJSON(w, dataUsageInfoJSON)
|
||||
}
|
||||
|
||||
func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquorum int) *madmin.LockEntry {
|
||||
func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
|
||||
entry := &madmin.LockEntry{
|
||||
Timestamp: l.Timestamp,
|
||||
Resource: resource,
|
||||
@ -356,18 +356,17 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquor
|
||||
Source: l.Source,
|
||||
Owner: l.Owner,
|
||||
ID: l.UID,
|
||||
Quorum: l.Quorum,
|
||||
}
|
||||
if l.Writer {
|
||||
entry.Type = "WRITE"
|
||||
entry.Quorum = wquorum
|
||||
} else {
|
||||
entry.Type = "READ"
|
||||
entry.Quorum = rquorum
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func topLockEntries(peerLocks []*PeerLocks, rquorum, wquorum int, stale bool) madmin.LockEntries {
|
||||
func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
|
||||
entryMap := make(map[string]*madmin.LockEntry)
|
||||
for _, peerLock := range peerLocks {
|
||||
if peerLock == nil {
|
||||
@ -379,7 +378,7 @@ func topLockEntries(peerLocks []*PeerLocks, rquorum, wquorum int, stale bool) ma
|
||||
if val, ok := entryMap[lockReqInfo.UID]; ok {
|
||||
val.ServerList = append(val.ServerList, peerLock.Addr)
|
||||
} else {
|
||||
entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr, rquorum, wquorum)
|
||||
entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -429,10 +428,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
|
||||
|
||||
peerLocks := globalNotificationSys.GetLocks(ctx, r)
|
||||
|
||||
rquorum := getReadQuorum(objectAPI.SetDriveCount())
|
||||
wquorum := getWriteQuorum(objectAPI.SetDriveCount())
|
||||
|
||||
topLocks := topLockEntries(peerLocks, rquorum, wquorum, stale)
|
||||
topLocks := topLockEntries(peerLocks, stale)
|
||||
|
||||
// Marshal API response upto requested count.
|
||||
if len(topLocks) > count && count > 0 {
|
||||
|
@ -178,7 +178,9 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
|
||||
}
|
||||
}
|
||||
|
||||
if len(disksInfo) == rootDiskCount {
|
||||
// Count offline disks as well to ensure consistent
|
||||
// reportability of offline drives on local setups.
|
||||
if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
|
||||
// Success.
|
||||
return disksInfo, errs, onlineDisks, offlineDisks
|
||||
}
|
||||
|
@ -35,6 +35,8 @@ type lockRequesterInfo struct {
|
||||
// Owner represents the UUID of the owner who originally requested the lock
|
||||
// useful in expiry.
|
||||
Owner string
|
||||
// Quorum represents the quorum required for this lock to be active.
|
||||
Quorum int
|
||||
}
|
||||
|
||||
// isWriteLock returns whether the lock is a write or read lock.
|
||||
@ -96,6 +98,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
|
||||
UID: args.UID,
|
||||
Timestamp: UTCNow(),
|
||||
TimeLastCheck: UTCNow(),
|
||||
Quorum: args.Quorum,
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -153,6 +156,7 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
|
||||
UID: args.UID,
|
||||
Timestamp: UTCNow(),
|
||||
TimeLastCheck: UTCNow(),
|
||||
Quorum: args.Quorum,
|
||||
}
|
||||
resource := args.Resources[0]
|
||||
if lri, ok := l.lockMap[resource]; ok {
|
||||
|
@ -23,6 +23,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
||||
"github.com/minio/minio/cmd/http"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
@ -93,6 +94,7 @@ func (client *lockRESTClient) restCall(ctx context.Context, call string, args ds
|
||||
values.Set(lockRESTUID, args.UID)
|
||||
values.Set(lockRESTOwner, args.Owner)
|
||||
values.Set(lockRESTSource, args.Source)
|
||||
values.Set(lockRESTQuorum, strconv.Itoa(args.Quorum))
|
||||
var buffer bytes.Buffer
|
||||
for _, resource := range args.Resources {
|
||||
buffer.WriteString(resource)
|
||||
|
@ -21,7 +21,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
lockRESTVersion = "v3"
|
||||
lockRESTVersion = "v4" // Add Quorum query param
|
||||
lockRESTVersionPrefix = SlashSeparator + lockRESTVersion
|
||||
lockRESTPrefix = minioReservedBucketPath + "/lock"
|
||||
)
|
||||
@ -43,6 +43,10 @@ const (
|
||||
// Source contains the line number, function and file name of the code
|
||||
// on the client node that requested the lock.
|
||||
lockRESTSource = "source"
|
||||
|
||||
// Quroum value to be saved along lock requester info, useful
|
||||
// in verifying stale locks
|
||||
lockRESTQuorum = "quorum"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"net/http"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@ -32,10 +33,10 @@ import (
|
||||
|
||||
const (
|
||||
// Lock maintenance interval.
|
||||
lockMaintenanceInterval = 30 * time.Second
|
||||
lockMaintenanceInterval = 15 * time.Second
|
||||
|
||||
// Lock validity check interval.
|
||||
lockValidityCheckInterval = 30 * time.Second
|
||||
lockValidityCheckInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// To abstract a node over network.
|
||||
@ -63,10 +64,16 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
|
||||
func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
|
||||
quorum, err := strconv.Atoi(r.URL.Query().Get(lockRESTQuorum))
|
||||
if err != nil {
|
||||
return args, err
|
||||
}
|
||||
|
||||
args = dsync.LockArgs{
|
||||
Owner: r.URL.Query().Get(lockRESTOwner),
|
||||
UID: r.URL.Query().Get(lockRESTUID),
|
||||
Source: r.URL.Query().Get(lockRESTSource),
|
||||
Quorum: quorum,
|
||||
}
|
||||
|
||||
var resources []string
|
||||
@ -277,12 +284,8 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
|
||||
for lendpoint, nlrips := range getLongLivedLocks(interval) {
|
||||
nlripsMap := make(map[string]nlock, len(nlrips))
|
||||
for _, nlrip := range nlrips {
|
||||
// Locks are only held on first zone, make sure that
|
||||
// we only look for ownership of locks from endpoints
|
||||
// on first zone.
|
||||
for _, c := range allLockersFn() {
|
||||
if !c.IsOnline() || c == nil {
|
||||
updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
|
||||
continue
|
||||
}
|
||||
|
||||
@ -306,14 +309,8 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Read locks we assume quorum for be N/2 success
|
||||
quorum := getReadQuorum(objAPI.SetDriveCount())
|
||||
if nlrip.lri.Writer {
|
||||
quorum = getWriteQuorum(objAPI.SetDriveCount())
|
||||
}
|
||||
|
||||
// less than the quorum, we have locks expired.
|
||||
if nlripsMap[nlrip.name].locks < quorum {
|
||||
if nlripsMap[nlrip.name].locks < nlrip.lri.Quorum {
|
||||
// The lock is no longer active at server that originated
|
||||
// the lock, attempt to remove the lock.
|
||||
globalLockServers[lendpoint].mutex.Lock()
|
||||
|
@ -241,6 +241,7 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
|
||||
UID: id,
|
||||
Resources: lockNames,
|
||||
Source: source,
|
||||
Quorum: quorum,
|
||||
}
|
||||
|
||||
var locked bool
|
||||
|
@ -33,6 +33,9 @@ type LockArgs struct {
|
||||
// Owner represents unique ID for this instance, an owner who originally requested
|
||||
// the locked resource, useful primarily in figuring our stale locks.
|
||||
Owner string
|
||||
|
||||
// Quorum represents the expected quorum for this lock type.
|
||||
Quorum int
|
||||
}
|
||||
|
||||
// NetLocker is dsync compatible locker interface.
|
||||
|
@ -38,7 +38,8 @@ type LockEntry struct {
|
||||
ServerList []string `json:"serverlist"` // List of servers participating in the lock.
|
||||
Owner string `json:"owner"` // Owner UUID indicates server owns the lock.
|
||||
ID string `json:"id"` // UID to uniquely identify request of client.
|
||||
Quorum int `json:"quorum"` // represents quorum number of servers required to hold this lock, used to look for stale locks.
|
||||
// Represents quorum number of servers required to hold this lock, used to look for stale locks.
|
||||
Quorum int `json:"quorum"`
|
||||
}
|
||||
|
||||
// LockEntries - To sort the locks
|
||||
|
Loading…
Reference in New Issue
Block a user