obd: Add console log to OBD output (#10372)

This commit is contained in:
Anis Elleuch 2020-09-16 02:02:54 +01:00 committed by GitHub
parent 80e3dce631
commit 8ea55f9dba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 130 additions and 25 deletions

View File

@ -1241,7 +1241,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
return return
} }
vars := mux.Vars(r) query := r.URL.Query()
obdInfo := madmin.OBDInfo{} obdInfo := madmin.OBDInfo{}
obdInfoCh := make(chan madmin.OBDInfo) obdInfoCh := make(chan madmin.OBDInfo)
@ -1287,7 +1287,13 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
go func() { go func() {
defer close(obdInfoCh) defer close(obdInfoCh)
if cpu, ok := vars["syscpu"]; ok && cpu == "true" { if log := query.Get("log"); log == "true" {
obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
partialWrite(obdInfo)
}
if cpu := query.Get("syscpu"); cpu == "true" {
cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r) cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo) obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
@ -1295,7 +1301,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" { if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r) diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo) obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
@ -1303,7 +1309,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" { if osInfo := query.Get("sysosinfo"); osInfo == "true" {
osInfo := getLocalOsInfoOBD(deadlinedCtx, r) osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo) obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
@ -1311,7 +1317,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if mem, ok := vars["sysmem"]; ok && mem == "true" { if mem := query.Get("sysmem"); mem == "true" {
memInfo := getLocalMemOBD(deadlinedCtx, r) memInfo := getLocalMemOBD(deadlinedCtx, r)
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo) obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
@ -1319,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if proc, ok := vars["sysprocess"]; ok && proc == "true" { if proc := query.Get("sysprocess"); proc == "true" {
procInfo := getLocalProcOBD(deadlinedCtx, r) procInfo := getLocalProcOBD(deadlinedCtx, r)
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo) obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
@ -1327,14 +1333,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if config, ok := vars["minioconfig"]; ok && config == "true" { if config := query.Get("minioconfig"); config == "true" {
cfg, err := readServerConfig(ctx, objectAPI) cfg, err := readServerConfig(ctx, objectAPI)
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
obdInfo.Minio.Config = cfg obdInfo.Minio.Config = cfg
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if drive, ok := vars["perfdrive"]; ok && drive == "true" { if drive := query.Get("perfdrive"); drive == "true" {
// Get drive obd details from local server's drive(s) // Get drive obd details from local server's drive(s)
driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r) driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r) driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
@ -1365,7 +1371,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure { if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx)) obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
partialWrite(obdInfo) partialWrite(obdInfo)
@ -1379,6 +1385,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx) obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
partialWrite(obdInfo) partialWrite(obdInfo)
} }
}() }()
ticker := time.NewTicker(30 * time.Second) ticker := time.NewTicker(30 * time.Second)

View File

@ -213,17 +213,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
if !globalIsGateway { if !globalIsGateway {
// -- OBD API -- // -- OBD API --
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo"). adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)). HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
Queries("perfdrive", "{perfdrive:true|false}",
"perfnet", "{perfnet:true|false}",
"minioinfo", "{minioinfo:true|false}",
"minioconfig", "{minioconfig:true|false}",
"syscpu", "{syscpu:true|false}",
"sysdiskhw", "{sysdiskhw:true|false}",
"sysosinfo", "{sysosinfo:true|false}",
"sysmem", "{sysmem:true|false}",
"sysprocess", "{sysprocess:true|false}",
)
} }
} }

View File

@ -43,10 +43,6 @@ func init() {
logger.Init(GOPATH, GOROOT) logger.Init(GOPATH, GOROOT)
logger.RegisterError(config.FmtError) logger.RegisterError(config.FmtError)
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
gob.Register(StorageErr("")) gob.Register(StorageErr(""))
} }

View File

@ -122,6 +122,24 @@ func (sys *HTTPConsoleLoggerSys) Validate() error {
return nil return nil
} }
// Content returns the console stdout log
func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
sys.RLock()
sys.logBuf.Do(func(p interface{}) {
if p != nil {
lg, ok := p.(log.Info)
if ok {
if (lg.Entry != log.Entry{}) {
logs = append(logs, lg.Entry)
}
}
}
})
sys.RUnlock()
return
}
// Send log message 'e' to console and publish to console // Send log message 'e' to console and publish to console
// log pubsub system // log pubsub system
func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error { func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

View File

@ -168,6 +168,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1) cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
} }
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
// Handle common command args. // Handle common command args.
handleCommonCmdArgs(ctx) handleCommonCmdArgs(ctx)

View File

@ -1118,6 +1118,36 @@ func (sys *NotificationSys) ProcOBDInfo(ctx context.Context) []madmin.ServerProc
return reply return reply
} }
// LogOBDInfo - Logs OBD information
func (sys *NotificationSys) LogOBDInfo(ctx context.Context) []madmin.ServerLogOBDInfo {
reply := make([]madmin.ServerLogOBDInfo, len(sys.peerClients))
g := errgroup.WithNErrs(len(sys.peerClients))
for index, client := range sys.peerClients {
if client == nil {
continue
}
index := index
g.Go(func() error {
var err error
reply[index], err = sys.peerClients[index].LogOBDInfo(ctx)
return err
}, index)
}
for index, err := range g.Wait() {
if err != nil {
addr := sys.peerClients[index].host.String()
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogIf(ctx, err)
reply[index].Addr = addr
reply[index].Error = err.Error()
}
}
return reply
}
// ServerInfo - calls ServerInfo RPC call on all peers. // ServerInfo - calls ServerInfo RPC call on all peers.
func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties { func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
reply := make([]madmin.ServerProperties, len(sys.peerClients)) reply := make([]madmin.ServerProperties, len(sys.peerClients))

View File

@ -148,6 +148,19 @@ func getLocalMemOBD(ctx context.Context, r *http.Request) madmin.ServerMemOBDInf
} }
} }
func getLocalLogOBD(ctx context.Context, r *http.Request) madmin.ServerLogOBDInfo {
addr := r.Host
if globalIsDistErasure {
addr = GetLocalPeer(globalEndpoints)
}
log := globalConsoleSys.Content()
return madmin.ServerLogOBDInfo{
Addr: addr,
Entries: log,
}
}
func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDInfo { func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDInfo {
addr := r.Host addr := r.Host
if globalIsDistErasure { if globalIsDistErasure {

View File

@ -420,6 +420,17 @@ func (client *peerRESTClient) ProcOBDInfo(ctx context.Context) (info madmin.Serv
return info, err return info, err
} }
// LogOBDInfo - fetch Log OBD information for a remote node.
func (client *peerRESTClient) LogOBDInfo(ctx context.Context) (info madmin.ServerLogOBDInfo, err error) {
respBody, err := client.callWithContext(ctx, peerRESTMethodLogOBDInfo, nil, nil, -1)
if err != nil {
return
}
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&info)
return info, err
}
// StartProfiling - Issues profiling command on the peer node. // StartProfiling - Issues profiling command on the peer node.
func (client *peerRESTClient) StartProfiling(profiler string) error { func (client *peerRESTClient) StartProfiling(profiler string) error {
values := make(url.Values) values := make(url.Values)

View File

@ -33,6 +33,7 @@ const (
peerRESTMethodOsInfoOBDInfo = "/osinfoobdinfo" peerRESTMethodOsInfoOBDInfo = "/osinfoobdinfo"
peerRESTMethodMemOBDInfo = "/memobdinfo" peerRESTMethodMemOBDInfo = "/memobdinfo"
peerRESTMethodProcOBDInfo = "/procobdinfo" peerRESTMethodProcOBDInfo = "/procobdinfo"
peerRESTMethodLogOBDInfo = "/logobdinfo"
peerRESTMethodDispatchNetOBDInfo = "/dispatchnetobdinfo" peerRESTMethodDispatchNetOBDInfo = "/dispatchnetobdinfo"
peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata" peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata"
peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata" peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata"

View File

@ -545,6 +545,22 @@ func (s *peerRESTServer) ProcOBDInfoHandler(w http.ResponseWriter, r *http.Reque
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info)) logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
} }
// LogOBDInfoHandler - returns Log OBD info.
func (s *peerRESTServer) LogOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
info := getLocalLogOBD(ctx, r)
defer w.(http.Flusher).Flush()
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
}
// MemOBDInfoHandler - returns Mem OBD info. // MemOBDInfoHandler - returns Mem OBD info.
func (s *peerRESTServer) MemOBDInfoHandler(w http.ResponseWriter, r *http.Request) { func (s *peerRESTServer) MemOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) { if !s.IsValid(w, r) {
@ -1039,6 +1055,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLogOBDInfo).HandlerFunc(httpTraceHdrs(server.LogOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemOBDInfo).HandlerFunc(httpTraceHdrs(server.MemOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemOBDInfo).HandlerFunc(httpTraceHdrs(server.MemOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfoOBDInfo).HandlerFunc(httpTraceHdrs(server.OsOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfoOBDInfo).HandlerFunc(httpTraceHdrs(server.OsOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwOBDInfo).HandlerFunc(httpTraceHdrs(server.DiskHwOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwOBDInfo).HandlerFunc(httpTraceHdrs(server.DiskHwOBDInfoHandler))

View File

@ -391,6 +391,7 @@ func serverMain(ctx *cli.Context) {
// Initialize globalConsoleSys system // Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext) globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
// Handle all server command args. // Handle all server command args.
serverHandleCmdArgs(ctx) serverHandleCmdArgs(ctx)

View File

@ -25,6 +25,7 @@ import (
"net/url" "net/url"
"time" "time"
"github.com/minio/minio/cmd/logger/message/log"
"github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/net" "github.com/minio/minio/pkg/net"
@ -42,6 +43,19 @@ type OBDInfo struct {
Perf PerfOBDInfo `json:"perf,omitempty"` Perf PerfOBDInfo `json:"perf,omitempty"`
Minio MinioOBDInfo `json:"minio,omitempty"` Minio MinioOBDInfo `json:"minio,omitempty"`
Sys SysOBDInfo `json:"sys,omitempty"` Sys SysOBDInfo `json:"sys,omitempty"`
Logging LogOBDInfo `json:"logging,omitempty"`
}
// LogOBDInfo holds different type of logs of all clusters
type LogOBDInfo struct {
ServersLog []ServerLogOBDInfo `json:"serverLogs"`
}
// ServerLogOBDInfo holds server's stdout log
type ServerLogOBDInfo struct {
Addr string `json:"nodeName"`
Entries []log.Entry `json:"entries"`
Error string `json:"error,omitempty"`
} }
// SysOBDInfo - Includes hardware and system information of the MinIO cluster // SysOBDInfo - Includes hardware and system information of the MinIO cluster
@ -185,6 +199,7 @@ const (
OBDDataTypeSysMem OBDDataType = "sysmem" OBDDataTypeSysMem OBDDataType = "sysmem"
OBDDataTypeSysNet OBDDataType = "sysnet" OBDDataTypeSysNet OBDDataType = "sysnet"
OBDDataTypeSysProcess OBDDataType = "sysprocess" OBDDataTypeSysProcess OBDDataType = "sysprocess"
OBDDataTypeLog OBDDataType = "log"
) )
// OBDDataTypesMap - Map of OBD datatypes // OBDDataTypesMap - Map of OBD datatypes
@ -201,6 +216,7 @@ var OBDDataTypesMap = map[string]OBDDataType{
"sysmem": OBDDataTypeSysMem, "sysmem": OBDDataTypeSysMem,
"sysnet": OBDDataTypeSysNet, "sysnet": OBDDataTypeSysNet,
"sysprocess": OBDDataTypeSysProcess, "sysprocess": OBDDataTypeSysProcess,
"log": OBDDataTypeLog,
} }
// OBDDataTypesList - List of OBD datatypes // OBDDataTypesList - List of OBD datatypes
@ -217,6 +233,7 @@ var OBDDataTypesList = []OBDDataType{
OBDDataTypeSysMem, OBDDataTypeSysMem,
OBDDataTypeSysNet, OBDDataTypeSysNet,
OBDDataTypeSysProcess, OBDDataTypeSysProcess,
OBDDataTypeLog,
} }
// ServerOBDInfo - Connect to a minio server and call OBD Info Management API // ServerOBDInfo - Connect to a minio server and call OBD Info Management API