Clean up lock-instrumentation and improve comments (#3499)

- Add a lockStat type to group counters
- Remove unnecessary helper functions
- Fix stats computation on force unlock
- Removed unnecessary checks and cleaned up comments
This commit is contained in:
Krishnan Parthasarathi 2016-12-26 23:59:55 +05:30 committed by Harshavardhana
parent e8ce3b64ed
commit 36fd317eb2
5 changed files with 176 additions and 180 deletions

View File

@ -17,7 +17,6 @@
package cmd package cmd
import ( import (
"errors"
"fmt" "fmt"
"time" "time"
) )
@ -36,59 +35,39 @@ const (
debugWLockStr lockType = "WLock" debugWLockStr lockType = "WLock"
) )
// Struct containing information of status (ready/running/blocked) of an operation with given operation ID. // debugLockInfo - represents a single lock's information, uniquely identified by opsID.
// See debugLockInfoPerVolumePath for more context.
type debugLockInfo struct { type debugLockInfo struct {
// "RLock" or "WLock". // "RLock" or "WLock".
lType lockType lType lockType
// Contains the trace of the function which invoked the lock, obtained from runtime. // Contains the backtrace of incl. the function which called (r)(un)lock.
lockSource string lockSource string
// Status can be running/ready/blocked. // Status can be running/blocked.
status statusType status statusType
// Time info of the since how long the status holds true. // Time of last status update.
since time.Time since time.Time
} }
// debugLockInfo - container for storing locking information for unique copy // debugLockInfoPerVolumePath - lock state information on all locks held on (volume, path).
// (volume,path) pair. ref variable holds the reference count for locks held for.
// `ref` values helps us understand the n locks held for given <volume, path> pair.
// `running` value helps us understand the total successful locks held (not blocked)
// for given <volume, path> pair and the operation is under execution. `blocked`
// value helps us understand the total number of operations blocked waiting on
// locks for given <volume,path> pair.
type debugLockInfoPerVolumePath struct { type debugLockInfoPerVolumePath struct {
ref int64 // running + blocked operations. counters *lockStat // Holds stats of lock held on (volume, path)
running int64 // count of successful lock acquire and running operations. lockInfo map[string]debugLockInfo // Lock information per operation ID.
blocked int64 // count of number of operations blocked waiting on lock.
lockInfo map[string]debugLockInfo // map of [opsID] debugLockInfo{operation, status, since} .
} }
// returns an instance of debugLockInfo. // LockInfoOriginMismatch - represents error when lock origin don't match.
// need to create this for every unique pair of {volume,path}. type LockInfoOriginMismatch struct {
// total locks, number of calls blocked on locks, and number of successful locks held but not unlocked yet.
func newDebugLockInfoPerVolumePath() *debugLockInfoPerVolumePath {
return &debugLockInfoPerVolumePath{
lockInfo: make(map[string]debugLockInfo),
ref: 0,
blocked: 0,
running: 0,
}
}
// LockInfoOriginNotFound - While changing the state of the lock info its important that the entry for
// lock at a given origin exists, if not `LockInfoOriginNotFound` is returned.
type LockInfoOriginNotFound struct {
volume string volume string
path string path string
opsID string opsID string
lockSource string lockSource string
} }
func (l LockInfoOriginNotFound) Error() string { func (l LockInfoOriginMismatch) Error() string {
return fmt.Sprintf("No lock state stored for the lock origined at \"%s\", for <volume> %s, <path> %s, <opsID> %s", return fmt.Sprintf("No lock state stored for the lock originated at \"%s\", for <volume> %s, <path> %s, <opsID> %s",
l.lockSource, l.volume, l.path, l.opsID) l.lockSource, l.volume, l.path, l.opsID)
} }
// LockInfoVolPathMissing - Error interface. Returned when the info the // LockInfoVolPathMissing - represents error when lock information is missing for a given (volume, path).
type LockInfoVolPathMissing struct { type LockInfoVolPathMissing struct {
volume string volume string
path string path string
@ -98,8 +77,7 @@ func (l LockInfoVolPathMissing) Error() string {
return fmt.Sprintf("No entry in debug Lock Map for Volume: %s, path: %s", l.volume, l.path) return fmt.Sprintf("No entry in debug Lock Map for Volume: %s, path: %s", l.volume, l.path)
} }
// LockInfoOpsIDNotFound - Returned when the lock state info exists, but the entry for // LockInfoOpsIDNotFound - represents error when lock info entry for a given operation ID doesn't exist.
// given operation ID doesn't exist.
type LockInfoOpsIDNotFound struct { type LockInfoOpsIDNotFound struct {
volume string volume string
path string path string
@ -110,8 +88,7 @@ func (l LockInfoOpsIDNotFound) Error() string {
return fmt.Sprintf("No entry in lock info for <Operation ID> %s, <volume> %s, <path> %s", l.opsID, l.volume, l.path) return fmt.Sprintf("No entry in lock info for <Operation ID> %s, <volume> %s, <path> %s", l.opsID, l.volume, l.path)
} }
// LockInfoStateNotBlocked - When an attempt to change the state of the lock form `blocked` to `running` is done, // LockInfoStateNotBlocked - represents error when lock info isn't in blocked state when it should be.
// its necessary that the state before the transsition is "blocked", otherwise LockInfoStateNotBlocked returned.
type LockInfoStateNotBlocked struct { type LockInfoStateNotBlocked struct {
volume string volume string
path string path string
@ -122,146 +99,126 @@ func (l LockInfoStateNotBlocked) Error() string {
return fmt.Sprintf("Lock state should be \"Blocked\" for <volume> %s, <path> %s, <opsID> %s", l.volume, l.path, l.opsID) return fmt.Sprintf("Lock state should be \"Blocked\" for <volume> %s, <path> %s, <opsID> %s", l.volume, l.path, l.opsID)
} }
var errLockNotInitialized = errors.New("Debug lockMap not initialized") // Initialize lock info for given (volume, path).
// Initialize lock info volume path.
func (n *nsLockMap) initLockInfoForVolumePath(param nsParam) { func (n *nsLockMap) initLockInfoForVolumePath(param nsParam) {
n.debugLockMap[param] = newDebugLockInfoPerVolumePath() n.debugLockMap[param] = &debugLockInfoPerVolumePath{
lockInfo: make(map[string]debugLockInfo),
counters: &lockStat{},
}
} }
// Change the state of the lock from Blocked to Running. // Change the state of the lock from Blocked to Running.
func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockSource, opsID string, readLock bool) error { func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockSource, opsID string, readLock bool) error {
// This operation is not executed under the scope nsLockMap.mutex.Lock(), lock has to be explicitly held here. // This function is called outside nsLockMap.mutex.Lock(), so must be held explicitly.
n.lockMapMutex.Lock() n.lockMapMutex.Lock()
defer n.lockMapMutex.Unlock() defer n.lockMapMutex.Unlock()
// new state info to be set for the lock.
newLockInfo := debugLockInfo{
lockSource: lockSource,
status: runningStatus,
since: time.Now().UTC(),
}
// Set lock type. // Check whether the lock info entry for <volume, path> pair already exists.
if readLock { _, ok := n.debugLockMap[param]
newLockInfo.lType = debugRLockStr
} else {
newLockInfo.lType = debugWLockStr
}
// Check whether the lock info entry for <volume, path> pair already exists and its not `nil`.
debugLockMap, ok := n.debugLockMap[param]
if !ok { if !ok {
// The lock state info foe given <volume, path> pair should already exist.
// If not return `LockInfoVolPathMissing`.
return traceError(LockInfoVolPathMissing{param.volume, param.path}) return traceError(LockInfoVolPathMissing{param.volume, param.path})
} }
// ``debugLockMap`` entry containing lock info for `param <volume, path>` is `nil`.
if debugLockMap == nil { // Check whether lock info entry for the given `opsID` exists.
return traceError(errLockNotInitialized)
}
lockInfo, ok := n.debugLockMap[param].lockInfo[opsID] lockInfo, ok := n.debugLockMap[param].lockInfo[opsID]
if !ok { if !ok {
// The lock info entry for given `opsID` should already exist for given <volume, path> pair.
// If not return `LockInfoOpsIDNotFound`.
return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID}) return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID})
} }
// The entry for the lock origined at `lockSource` should already exist. If not return `LockInfoOriginNotFound`.
// Check whether lockSource is same.
if lockInfo.lockSource != lockSource { if lockInfo.lockSource != lockSource {
return traceError(LockInfoOriginNotFound{param.volume, param.path, opsID, lockSource}) return traceError(LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource})
} }
// Status of the lock should already be set to "Blocked". If not return `LockInfoStateNotBlocked`.
// Status of the lock should be set to "Blocked".
if lockInfo.status != blockedStatus { if lockInfo.status != blockedStatus {
return traceError(LockInfoStateNotBlocked{param.volume, param.path, opsID}) return traceError(LockInfoStateNotBlocked{param.volume, param.path, opsID})
} }
// All checks finished. Changing the status of the operation from blocked to running and updating the time. // Change lock status to running and update the time.
n.debugLockMap[param].lockInfo[opsID] = newLockInfo n.debugLockMap[param].lockInfo[opsID] = newDebugLockInfo(lockSource, runningStatus, readLock)
// After locking unblocks decrease the blocked counter. // Update global lock stats.
n.blockedCounter-- n.counters.lockGranted()
// Increase the running counter. // Update (volume, pair) lock stats.
n.runningLockCounter++ n.debugLockMap[param].counters.lockGranted()
n.debugLockMap[param].blocked--
n.debugLockMap[param].running++
return nil return nil
} }
// Change the state of the lock from Ready to Blocked. // newDebugLockInfo - Constructs a debugLockInfo value given lock source, status and type.
func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockSource, opsID string, readLock bool) error { func newDebugLockInfo(lockSource string, status statusType, readLock bool) debugLockInfo {
newLockInfo := debugLockInfo{ lType := debugRLockStr
if readLock {
lType = debugRLockStr
} else {
lType = debugWLockStr
}
return debugLockInfo{
lockSource: lockSource, lockSource: lockSource,
status: blockedStatus, lType: lType,
status: status,
since: time.Now().UTC(), since: time.Now().UTC(),
} }
if readLock {
newLockInfo.lType = debugRLockStr
} else {
newLockInfo.lType = debugWLockStr
} }
lockInfo, ok := n.debugLockMap[param] // Change the state of the lock to Blocked.
func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockSource, opsID string, readLock bool) error {
_, ok := n.debugLockMap[param]
if !ok { if !ok {
// State info entry for the given <volume, pair> doesn't exist, initializing it. // Lock info entry for (volume, pair) doesn't exist, initialize it.
n.initLockInfoForVolumePath(param)
}
if lockInfo == nil {
// *lockInfo is nil, initialize here.
n.initLockInfoForVolumePath(param) n.initLockInfoForVolumePath(param)
} }
// lockInfo is a map[string]debugLockInfo, which holds map[OperationID]{status,time, origin} of the lock. // Mark lock status blocked for given opsID.
if n.debugLockMap[param].lockInfo == nil { n.debugLockMap[param].lockInfo[opsID] = newDebugLockInfo(lockSource, blockedStatus, readLock)
n.debugLockMap[param].lockInfo = make(map[string]debugLockInfo) // Update global lock stats.
} n.counters.lockWaiting()
// The status of the operation with the given operation ID is marked blocked till its gets unblocked from the lock. // Update (volume, path) lock stats.
n.debugLockMap[param].lockInfo[opsID] = newLockInfo n.debugLockMap[param].counters.lockWaiting()
// Increment the Global lock counter.
n.globalLockCounter++
// Increment the counter for number of blocked opertions, decrement it after the locking unblocks.
n.blockedCounter++
// increment the reference of the lock for the given <volume,path> pair.
n.debugLockMap[param].ref++
// increment the blocked counter for the given <volume, path> pair.
n.debugLockMap[param].blocked++
return nil return nil
} }
// deleteLockInfoEntry - Deletes the lock state information for given // deleteLockInfoEntry - Deletes the lock information for given (volume, path).
// <volume, path> pair. Called when nsLk.ref count is 0. // Called when nsLk.ref count is 0.
func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error { func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error {
// delete the lock info for the given operation. // delete the lock info for the given operation.
if _, found := n.debugLockMap[param]; !found { if _, found := n.debugLockMap[param]; !found {
return traceError(LockInfoVolPathMissing{param.volume, param.path}) return traceError(LockInfoVolPathMissing{param.volume, param.path})
} }
// Remove from the map if there are no more references for the given (volume,path) pair.
// The following stats update is relevant only in case of a
// ForceUnlock. In case of the last unlock on a (volume,
// path), this would be a no-op.
volumePathLocks := n.debugLockMap[param]
for _, lockInfo := range volumePathLocks.lockInfo {
granted := lockInfo.status == runningStatus
// Update global and (volume, path) stats.
n.counters.lockRemoved(granted)
volumePathLocks.counters.lockRemoved(granted)
}
delete(n.debugLockMap, param) delete(n.debugLockMap, param)
return nil return nil
} }
// deleteLockInfoEntry - Deletes the entry for given opsID in the lock state information // deleteLockInfoEntry - Deletes lock info entry for given opsID.
// of given <volume, path> pair. Called when the nsLk ref count for the given // Called when the nsLk ref count for the given (volume, path) is
// <volume, path> pair is not 0. // not 0.
func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error { func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error {
// delete the lock info for the given operation. // delete the lock info for the given operation.
infoMap, found := n.debugLockMap[param] infoMap, found := n.debugLockMap[param]
if !found { if !found {
return traceError(LockInfoVolPathMissing{param.volume, param.path}) return traceError(LockInfoVolPathMissing{param.volume, param.path})
} }
// The opertion finished holding the lock on the resource, remove // The operation finished holding the lock on the resource, remove
// the entry for the given operation with the operation ID. // the entry for the given operation with the operation ID.
_, foundInfo := infoMap.lockInfo[opsID] opsIDLock, foundInfo := infoMap.lockInfo[opsID]
if !foundInfo { if !foundInfo {
// Unlock request with invalid opertion ID not accepted. // Unlock request with invalid operation ID not accepted.
return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID}) return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID})
} }
// Decrease the global running and lock reference counter. // Update global and (volume, path) lock status.
n.runningLockCounter-- granted := opsIDLock.status == runningStatus
n.globalLockCounter-- n.counters.lockRemoved(granted)
// Decrease the lock referee counter for the lock info for given <volume,path> pair. infoMap.counters.lockRemoved(granted)
// Decrease the running operation number. Its assumed that the operation is over
// once an attempt to release the lock is made.
infoMap.running--
// Decrease the total reference count of locks jeld on <volume,path> pair.
infoMap.ref--
delete(infoMap.lockInfo, opsID) delete(infoMap.lockInfo, opsID)
return nil return nil
} }

View File

@ -124,19 +124,19 @@ func verifyGlobalLockStats(l lockStateCase, t *testing.T, testNum int) {
globalNSMutex.lockMapMutex.Lock() globalNSMutex.lockMapMutex.Lock()
// Verifying the lock stats. // Verifying the lock stats.
if globalNSMutex.globalLockCounter != int64(l.expectedGlobalLockCount) { if globalNSMutex.counters.total != int64(l.expectedGlobalLockCount) {
t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount), t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount),
globalNSMutex.globalLockCounter) globalNSMutex.counters.total)
} }
// verify the count for total blocked locks. // verify the count for total blocked locks.
if globalNSMutex.blockedCounter != int64(l.expectedBlockedLockCount) { if globalNSMutex.counters.blocked != int64(l.expectedBlockedLockCount) {
t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount), t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount),
globalNSMutex.blockedCounter) globalNSMutex.counters.blocked)
} }
// verify the count for total running locks. // verify the count for total running locks.
if globalNSMutex.runningLockCounter != int64(l.expectedRunningLockCount) { if globalNSMutex.counters.granted != int64(l.expectedRunningLockCount) {
t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount), t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount),
globalNSMutex.runningLockCounter) globalNSMutex.counters.granted)
} }
globalNSMutex.lockMapMutex.Unlock() globalNSMutex.lockMapMutex.Unlock()
// Verifying again with the JSON response of the lock info. // Verifying again with the JSON response of the lock info.
@ -169,19 +169,19 @@ func verifyLockStats(l lockStateCase, t *testing.T, testNum int) {
param := nsParam{l.volume, l.path} param := nsParam{l.volume, l.path}
// Verify the total locks (blocked+running) for given <vol,path> pair. // Verify the total locks (blocked+running) for given <vol,path> pair.
if globalNSMutex.debugLockMap[param].ref != int64(l.expectedVolPathLockCount) { if globalNSMutex.debugLockMap[param].counters.total != int64(l.expectedVolPathLockCount) {
t.Errorf("Test %d: Expected the total lock count for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, t.Errorf("Test %d: Expected the total lock count for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum,
param.volume, param.path, int64(l.expectedVolPathLockCount), globalNSMutex.debugLockMap[param].ref) param.volume, param.path, int64(l.expectedVolPathLockCount), globalNSMutex.debugLockMap[param].counters.total)
} }
// Verify the total running locks for given <volume, path> pair. // Verify the total running locks for given <volume, path> pair.
if globalNSMutex.debugLockMap[param].running != int64(l.expectedVolPathRunningCount) { if globalNSMutex.debugLockMap[param].counters.granted != int64(l.expectedVolPathRunningCount) {
t.Errorf("Test %d: Expected the total running locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path, t.Errorf("Test %d: Expected the total running locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path,
int64(l.expectedVolPathRunningCount), globalNSMutex.debugLockMap[param].running) int64(l.expectedVolPathRunningCount), globalNSMutex.debugLockMap[param].counters.granted)
} }
// Verify the total blocked locks for givne <volume, path> pair. // Verify the total blocked locks for givne <volume, path> pair.
if globalNSMutex.debugLockMap[param].blocked != int64(l.expectedVolPathBlockCount) { if globalNSMutex.debugLockMap[param].counters.blocked != int64(l.expectedVolPathBlockCount) {
t.Errorf("Test %d: Expected the total blocked locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path, t.Errorf("Test %d: Expected the total blocked locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path,
int64(l.expectedVolPathBlockCount), globalNSMutex.debugLockMap[param].blocked) int64(l.expectedVolPathBlockCount), globalNSMutex.debugLockMap[param].counters.blocked)
} }
} }
@ -230,16 +230,19 @@ func verifyLockState(l lockStateCase, t *testing.T, testNum int) {
// TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath(). // TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath().
func TestNewDebugLockInfoPerVolumePath(t *testing.T) { func TestNewDebugLockInfoPerVolumePath(t *testing.T) {
lockInfo := newDebugLockInfoPerVolumePath() lockInfo := &debugLockInfoPerVolumePath{
lockInfo: make(map[string]debugLockInfo),
counters: &lockStat{},
}
if lockInfo.ref != 0 { if lockInfo.counters.total != 0 {
t.Errorf("Expected initial reference value of total locks to be 0, got %d", lockInfo.ref) t.Errorf("Expected initial reference value of total locks to be 0, got %d", lockInfo.counters.total)
} }
if lockInfo.blocked != 0 { if lockInfo.counters.blocked != 0 {
t.Errorf("Expected initial reference of blocked locks to be 0, got %d", lockInfo.blocked) t.Errorf("Expected initial reference of blocked locks to be 0, got %d", lockInfo.counters.blocked)
} }
if lockInfo.running != 0 { if lockInfo.counters.granted != 0 {
t.Errorf("Expected initial reference value of held locks to be 0, got %d", lockInfo.running) t.Errorf("Expected initial reference value of held locks to be 0, got %d", lockInfo.counters.granted)
} }
} }
@ -300,7 +303,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
readLock: true, readLock: true,
setBlocked: false, setBlocked: false,
// expected metrics. // expected metrics.
expectedErr: LockInfoOriginNotFound{"my-bucket", "my-object", "abcd1234", "Bad Origin"}, expectedErr: LockInfoOriginMismatch{"my-bucket", "my-object", "abcd1234", "Bad Origin"},
}, },
// Test case - 5. // Test case - 5.
// Test case with write lock. // Test case with write lock.
@ -332,21 +335,11 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
debugLockMap: make(map[nsParam]*debugLockInfoPerVolumePath), debugLockMap: make(map[nsParam]*debugLockInfoPerVolumePath),
lockMap: make(map[nsParam]*nsLock), lockMap: make(map[nsParam]*nsLock),
} }
// Entry for <volume, path> pair is set to nil. Should fail with `errLockNotInitialized`.
globalNSMutex.debugLockMap[param] = nil
actualErr = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
testCases[0].opsID, testCases[0].readLock)
if errorCause(actualErr) != errLockNotInitialized {
t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", errLockNotInitialized, actualErr)
}
// Setting the lock info the be `nil`. // Setting the lock info the be `nil`.
globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{ globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{
lockInfo: nil, // setting the lockinfo to nil. lockInfo: nil, // setting the lockinfo to nil.
ref: 0, counters: &lockStat{},
blocked: 0,
running: 0,
} }
actualErr = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource, actualErr = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
@ -361,9 +354,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
// but the initial state if already "Running". Such an attempt should fail // but the initial state if already "Running". Such an attempt should fail
globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{ globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{
lockInfo: make(map[string]debugLockInfo), lockInfo: make(map[string]debugLockInfo),
ref: 0, counters: &lockStat{},
blocked: 0,
running: 0,
} }
// Setting the status of the lock to be "Running". // Setting the status of the lock to be "Running".
@ -610,14 +601,14 @@ func TestNsLockMapDeleteLockInfoEntryForOps(t *testing.T) {
} else { } else {
t.Fatalf("Entry for <volume> %s, <path> %s should have existed. ", param.volume, param.path) t.Fatalf("Entry for <volume> %s, <path> %s should have existed. ", param.volume, param.path)
} }
if globalNSMutex.runningLockCounter != int64(0) { if globalNSMutex.counters.granted != int64(0) {
t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), globalNSMutex.runningLockCounter) t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), globalNSMutex.counters.granted)
} }
if globalNSMutex.blockedCounter != int64(0) { if globalNSMutex.counters.blocked != int64(0) {
t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), globalNSMutex.blockedCounter) t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), globalNSMutex.counters.blocked)
} }
if globalNSMutex.globalLockCounter != int64(0) { if globalNSMutex.counters.total != int64(0) {
t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), globalNSMutex.globalLockCounter) t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), globalNSMutex.counters.total)
} }
} }
@ -680,13 +671,13 @@ func TestNsLockMapDeleteLockInfoEntryForVolumePath(t *testing.T) {
t.Fatalf("Entry for <volume> %s, <path> %s should have been deleted. ", param.volume, param.path) t.Fatalf("Entry for <volume> %s, <path> %s should have been deleted. ", param.volume, param.path)
} }
// The lock count values should be 0. // The lock count values should be 0.
if globalNSMutex.runningLockCounter != int64(0) { if globalNSMutex.counters.granted != int64(0) {
t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), globalNSMutex.runningLockCounter) t.Errorf("Expected the count of total running locks to be %v, but got %v", int64(0), globalNSMutex.counters.granted)
} }
if globalNSMutex.blockedCounter != int64(0) { if globalNSMutex.counters.blocked != int64(0) {
t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), globalNSMutex.blockedCounter) t.Errorf("Expected the count of total blocked locks to be %v, but got %v", int64(0), globalNSMutex.counters.blocked)
} }
if globalNSMutex.globalLockCounter != int64(0) { if globalNSMutex.counters.total != int64(0) {
t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), globalNSMutex.globalLockCounter) t.Errorf("Expected the count of all locks to be %v, but got %v", int64(0), globalNSMutex.counters.total)
} }
} }

49
cmd/lock-stat.go Normal file
View File

@ -0,0 +1,49 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
// lockStat - encapsulates total, blocked and granted lock counts.
type lockStat struct {
total int64
blocked int64
granted int64
}
// lockWaiting - updates lock stat when a lock becomes blocked.
func (ls *lockStat) lockWaiting() {
ls.blocked++
ls.total++
}
// lockGranted - updates lock stat when a lock is granted.
func (ls *lockStat) lockGranted() {
ls.blocked--
ls.granted++
}
// lockRemoved - updates lock stat when a lock is removed, by Unlock
// or ForceUnlock.
func (ls *lockStat) lockRemoved(granted bool) {
if granted {
ls.granted--
ls.total--
} else {
ls.blocked--
ls.total--
}
}

View File

@ -66,17 +66,17 @@ func getSystemLockState() (SystemLockState, error) {
lockState := SystemLockState{} lockState := SystemLockState{}
lockState.TotalBlockedLocks = globalNSMutex.blockedCounter lockState.TotalBlockedLocks = globalNSMutex.counters.blocked
lockState.TotalLocks = globalNSMutex.globalLockCounter lockState.TotalLocks = globalNSMutex.counters.total
lockState.TotalAcquiredLocks = globalNSMutex.runningLockCounter lockState.TotalAcquiredLocks = globalNSMutex.counters.granted
for param, debugLock := range globalNSMutex.debugLockMap { for param, debugLock := range globalNSMutex.debugLockMap {
volLockInfo := VolumeLockInfo{} volLockInfo := VolumeLockInfo{}
volLockInfo.Bucket = param.volume volLockInfo.Bucket = param.volume
volLockInfo.Object = param.path volLockInfo.Object = param.path
volLockInfo.LocksOnObject = debugLock.ref volLockInfo.LocksOnObject = debugLock.counters.total
volLockInfo.TotalBlockedLocks = debugLock.blocked volLockInfo.TotalBlockedLocks = debugLock.counters.blocked
volLockInfo.LocksAcquiredOnObject = debugLock.running volLockInfo.LocksAcquiredOnObject = debugLock.counters.granted
for opsID, lockInfo := range debugLock.lockInfo { for opsID, lockInfo := range debugLock.lockInfo {
volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject, OpsLockState{ volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject, OpsLockState{
OperationID: opsID, OperationID: opsID,

View File

@ -61,6 +61,7 @@ func initNSLock(isDistXL bool) {
globalNSMutex = &nsLockMap{ globalNSMutex = &nsLockMap{
isDistXL: isDistXL, isDistXL: isDistXL,
lockMap: make(map[nsParam]*nsLock), lockMap: make(map[nsParam]*nsLock),
counters: &lockStat{},
} }
// Initialize nsLockMap with entry for instrumentation information. // Initialize nsLockMap with entry for instrumentation information.
@ -91,9 +92,7 @@ type nsLock struct {
// Unlock, RLock and RUnlock. // Unlock, RLock and RUnlock.
type nsLockMap struct { type nsLockMap struct {
// Lock counter used for lock debugging. // Lock counter used for lock debugging.
globalLockCounter int64 // Total locks held. counters *lockStat
blockedCounter int64 // Total operations blocked waiting for locks.
runningLockCounter int64 // Total locks held but not released yet.
debugLockMap map[nsParam]*debugLockInfoPerVolumePath // Info for instrumentation on locks. debugLockMap map[nsParam]*debugLockInfoPerVolumePath // Info for instrumentation on locks.
// Indicates whether the locking service is part // Indicates whether the locking service is part