mirror of
https://github.com/minio/minio.git
synced 2025-11-29 05:19:03 -05:00
lock/instrumentation: Cleanup and print in user friendly form. (#2807)
This commit is contained in:
Harshavardhana
GitHub
@@ -17,129 +17,134 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/mc/pkg/console"
|
||||
)
|
||||
|
||||
// SystemLockState - Structure to fill the lock state of entire object storage.
|
||||
// That is the total locks held, total calls blocked on locks and state of all the locks for the entire system.
|
||||
type SystemLockState struct {
|
||||
TotalLocks int64 `json:"totalLocks"`
|
||||
TotalBlockedLocks int64 `json:"totalBlockedLocks"` // count of operations which are blocked waiting for the lock to be released.
|
||||
TotalAcquiredLocks int64 `json:"totalAcquiredLocks"` // count of operations which has successfully acquired the lock but hasn't unlocked yet( operation in progress).
|
||||
LocksInfoPerObject []VolumeLockInfo `json:"locksInfoPerObject"`
|
||||
}
|
||||
|
||||
// VolumeLockInfo - Structure to contain the lock state info for volume, path pair.
|
||||
type VolumeLockInfo struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Object string `json:"object"`
|
||||
LocksOnObject int64 `json:"locksOnObject"` // All locks blocked + running for given <volume,path> pair.
|
||||
LocksAcquiredOnObject int64 `json:"locksAcquiredOnObject"` // count of operations which has successfully acquired the lock but hasn't unlocked yet( operation in progress).
|
||||
TotalBlockedLocks int64 `json:"locksBlockedOnObject"` // count of operations which are blocked waiting for the lock to be released.
|
||||
LockDetailsOnObject []OpsLockState `json:"lockDetailsOnObject"` // state information containing state of the locks for all operations on given <volume,path> pair.
|
||||
}
|
||||
|
||||
// OpsLockState - structure to fill in state information of the lock.
|
||||
// structure to fill in status information for each operation with given operation ID.
|
||||
type OpsLockState struct {
|
||||
OperationID string `json:"opsID"` // string containing operation ID.
|
||||
LockOrigin string `json:"lockOrigin"` // contant which mentions the operation type (Get Obejct, PutObject...)
|
||||
LockType string `json:"lockType"`
|
||||
Status string `json:"status"` // status can be running/ready/blocked.
|
||||
StatusSince string `json:"statusSince"` // time info of the since how long the status holds true, value in seconds.
|
||||
}
|
||||
|
||||
// Read entire state of the locks in the system and return.
|
||||
func generateSystemLockResponse() (SystemLockState, error) {
|
||||
nsMutex.lockMapMutex.Lock()
|
||||
defer nsMutex.lockMapMutex.Unlock()
|
||||
|
||||
if nsMutex.debugLockMap == nil {
|
||||
return SystemLockState{}, errLockNotInitialized
|
||||
}
|
||||
|
||||
lockState := SystemLockState{}
|
||||
|
||||
lockState.TotalBlockedLocks = nsMutex.blockedCounter
|
||||
lockState.TotalLocks = nsMutex.globalLockCounter
|
||||
lockState.TotalAcquiredLocks = nsMutex.runningLockCounter
|
||||
|
||||
for param := range nsMutex.debugLockMap {
|
||||
volLockInfo := VolumeLockInfo{}
|
||||
volLockInfo.Bucket = param.volume
|
||||
volLockInfo.Object = param.path
|
||||
volLockInfo.TotalBlockedLocks = nsMutex.debugLockMap[param].blocked
|
||||
volLockInfo.LocksAcquiredOnObject = nsMutex.debugLockMap[param].running
|
||||
volLockInfo.LocksOnObject = nsMutex.debugLockMap[param].ref
|
||||
for opsID := range nsMutex.debugLockMap[param].lockInfo {
|
||||
opsState := OpsLockState{}
|
||||
opsState.OperationID = opsID
|
||||
opsState.LockOrigin = nsMutex.debugLockMap[param].lockInfo[opsID].lockOrigin
|
||||
opsState.LockType = nsMutex.debugLockMap[param].lockInfo[opsID].lockType
|
||||
opsState.Status = nsMutex.debugLockMap[param].lockInfo[opsID].status
|
||||
opsState.StatusSince = time.Now().UTC().Sub(nsMutex.debugLockMap[param].lockInfo[opsID].since).String()
|
||||
|
||||
volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject, opsState)
|
||||
}
|
||||
lockState.LocksInfoPerObject = append(lockState.LocksInfoPerObject, volLockInfo)
|
||||
}
|
||||
|
||||
return lockState, nil
|
||||
var lockFlags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "older-than",
|
||||
Usage: "List locks older than given time.",
|
||||
Value: "24h",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
Usage: "Lists more information about locks.",
|
||||
},
|
||||
}
|
||||
|
||||
var lockCmd = cli.Command{
|
||||
Name: "lock",
|
||||
Usage: "info about the locks in the node.",
|
||||
Usage: "Prints current lock information.",
|
||||
Action: lockControl,
|
||||
Flags: globalFlags,
|
||||
Flags: append(lockFlags, globalFlags...),
|
||||
CustomHelpTemplate: `NAME:
|
||||
minio control {{.Name}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
minio control {{.Name}} http://localhost:9000/
|
||||
minio control {{.Name}} [list|clear] http://localhost:9000/
|
||||
|
||||
FLAGS:
|
||||
{{range .Flags}}{{.}}
|
||||
{{end}}
|
||||
|
||||
EAMPLES:
|
||||
1. Get all the info about the blocked/held locks in the node:
|
||||
$ minio control lock http://localhost:9000/
|
||||
1. List all currently active locks from all nodes. Defaults to list locks held longer than 24hrs.
|
||||
$ minio control {{.Name}} list http://localhost:9000/
|
||||
|
||||
2. List all currently active locks from all nodes. Request locks from older than 1minute.
|
||||
$ minio control {{.Name}} --older-than=1m list http://localhost:9000/
|
||||
`,
|
||||
}
|
||||
|
||||
// printLockStateVerbose - pretty prints systemLockState, additionally this filters out based on a given duration.
|
||||
func printLockStateVerbose(lkStateRep map[string]SystemLockState, olderThan time.Duration) {
|
||||
console.Println("Duration Server LockType LockAcquired Status LockOrigin Resource")
|
||||
for server, lockState := range lkStateRep {
|
||||
for _, lockInfo := range lockState.LocksInfoPerObject {
|
||||
lockedResource := path.Join(lockInfo.Bucket, lockInfo.Object)
|
||||
for _, lockDetails := range lockInfo.LockDetailsOnObject {
|
||||
if lockDetails.Duration < olderThan {
|
||||
continue
|
||||
}
|
||||
console.Println(lockDetails.Duration, server,
|
||||
lockDetails.LockType, lockDetails.Since,
|
||||
lockDetails.Status, lockDetails.LockOrigin,
|
||||
lockedResource)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printLockState - pretty prints systemLockState, additionally this filters out based on a given duration.
|
||||
func printLockState(lkStateRep map[string]SystemLockState, olderThan time.Duration) {
|
||||
console.Println("Duration Server LockType Resource")
|
||||
for server, lockState := range lkStateRep {
|
||||
for _, lockInfo := range lockState.LocksInfoPerObject {
|
||||
lockedResource := path.Join(lockInfo.Bucket, lockInfo.Object)
|
||||
for _, lockDetails := range lockInfo.LockDetailsOnObject {
|
||||
if lockDetails.Duration < olderThan {
|
||||
continue
|
||||
}
|
||||
console.Println(lockDetails.Duration, server,
|
||||
lockDetails.LockType, lockedResource)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// "minio control lock" entry point.
|
||||
func lockControl(c *cli.Context) {
|
||||
if len(c.Args()) != 1 {
|
||||
if !c.Args().Present() && len(c.Args()) != 2 {
|
||||
cli.ShowCommandHelpAndExit(c, "lock", 1)
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(c.Args()[0])
|
||||
parsedURL, err := url.Parse(c.Args().Get(1))
|
||||
fatalIf(err, "Unable to parse URL.")
|
||||
|
||||
// Parse older than string.
|
||||
olderThanStr := c.String("older-than")
|
||||
olderThan, err := time.ParseDuration(olderThanStr)
|
||||
fatalIf(err, "Unable to parse older-than time duration.")
|
||||
|
||||
// Verbose flag.
|
||||
verbose := c.Bool("verbose")
|
||||
|
||||
authCfg := &authConfig{
|
||||
accessKey: serverConfig.GetCredential().AccessKeyID,
|
||||
secretKey: serverConfig.GetCredential().SecretAccessKey,
|
||||
secureConn: parsedURL.Scheme == "https",
|
||||
address: parsedURL.Host,
|
||||
path: path.Join(reservedBucket, controlPath),
|
||||
loginMethod: "Controller.LoginHandler",
|
||||
loginMethod: "Control.LoginHandler",
|
||||
}
|
||||
client := newAuthClient(authCfg)
|
||||
|
||||
args := &GenericArgs{}
|
||||
reply := &SystemLockState{}
|
||||
err = client.Call("Controller.LockInfo", args, reply)
|
||||
// logs the error and returns if err != nil.
|
||||
fatalIf(err, "RPC Controller.LockInfo call failed")
|
||||
// print the lock info on the console.
|
||||
b, err := json.MarshalIndent(*reply, "", " ")
|
||||
fatalIf(err, "Failed to parse the RPC lock info response")
|
||||
fmt.Print(string(b))
|
||||
args := &GenericArgs{
|
||||
// This is necessary so that the remotes,
|
||||
// don't end up sending requests back and forth.
|
||||
Remote: true,
|
||||
}
|
||||
|
||||
subCommand := c.Args().Get(0)
|
||||
switch subCommand {
|
||||
case "list":
|
||||
lkStateRep := make(map[string]SystemLockState)
|
||||
// Request lock info, fetches from all the nodes in the cluster.
|
||||
err = client.Call("Control.LockInfo", args, &lkStateRep)
|
||||
fatalIf(err, "Unable to fetch system lockInfo.")
|
||||
if !verbose {
|
||||
printLockState(lkStateRep, olderThan)
|
||||
} else {
|
||||
printLockStateVerbose(lkStateRep, olderThan)
|
||||
}
|
||||
case "clear":
|
||||
// TODO. Defaults to clearing all locks.
|
||||
default:
|
||||
fatalIf(errInvalidArgument, "Unsupported lock control operation %s", c.Args().Get(0))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user