From 8ea55f9dba33bca283f94166b84ec3c95dcd964b Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Wed, 16 Sep 2020 02:02:54 +0100 Subject: [PATCH] obd: Add console log to OBD output (#10372) --- cmd/admin-handlers.go | 25 ++++++++++++++++--------- cmd/admin-router.go | 14 ++------------ cmd/common-main.go | 4 ---- cmd/consolelogger.go | 18 ++++++++++++++++++ cmd/gateway-main.go | 4 ++++ cmd/notification.go | 30 ++++++++++++++++++++++++++++++ cmd/obdinfo.go | 13 +++++++++++++ cmd/peer-rest-client.go | 11 +++++++++++ cmd/peer-rest-common.go | 1 + cmd/peer-rest-server.go | 17 +++++++++++++++++ cmd/server-main.go | 1 + pkg/madmin/obd.go | 17 +++++++++++++++++ 12 files changed, 130 insertions(+), 25 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 2814f1520..26e6367da 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -1241,7 +1241,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) return } - vars := mux.Vars(r) + query := r.URL.Query() obdInfo := madmin.OBDInfo{} obdInfoCh := make(chan madmin.OBDInfo) @@ -1287,7 +1287,13 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) go func() { defer close(obdInfoCh) - if cpu, ok := vars["syscpu"]; ok && cpu == "true" { + if log := query.Get("log"); log == "true" { + obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r)) + obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...) + partialWrite(obdInfo) + } + + if cpu := query.Get("syscpu"); cpu == "true" { cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r) obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo) @@ -1295,7 +1301,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" { + if diskHw := query.Get("sysdiskhw"); diskHw == "true" { diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r) obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo) @@ -1303,7 +1309,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" { + if osInfo := query.Get("sysosinfo"); osInfo == "true" { osInfo := getLocalOsInfoOBD(deadlinedCtx, r) obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo) @@ -1311,7 +1317,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if mem, ok := vars["sysmem"]; ok && mem == "true" { + if mem := query.Get("sysmem"); mem == "true" { memInfo := getLocalMemOBD(deadlinedCtx, r) obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo) @@ -1319,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if proc, ok := vars["sysprocess"]; ok && proc == "true" { + if proc := query.Get("sysprocess"); proc == "true" { procInfo := getLocalProcOBD(deadlinedCtx, r) obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo) @@ -1327,14 +1333,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if config, ok := vars["minioconfig"]; ok && config == "true" { + if config := query.Get("minioconfig"); config == "true" { cfg, err := readServerConfig(ctx, objectAPI) logger.LogIf(ctx, err) obdInfo.Minio.Config = cfg partialWrite(obdInfo) } - if drive, ok := vars["perfdrive"]; ok && drive == "true" { + if drive := query.Get("perfdrive"); drive == "true" { // Get drive obd details from local server's drive(s) driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r) driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r) @@ -1365,7 +1371,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) partialWrite(obdInfo) } - if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure { + if net := query.Get("perfnet"); net == "true" && globalIsDistErasure { obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx)) partialWrite(obdInfo) @@ -1379,6 +1385,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx) partialWrite(obdInfo) } + }() ticker := time.NewTicker(30 * time.Second) diff --git a/cmd/admin-router.go b/cmd/admin-router.go index f6c72296e..c5a74dfdf 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -212,18 +212,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) if !globalIsGateway { // -- OBD API -- - adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo"). - HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)). - Queries("perfdrive", "{perfdrive:true|false}", - "perfnet", "{perfnet:true|false}", - "minioinfo", "{minioinfo:true|false}", - "minioconfig", "{minioconfig:true|false}", - "syscpu", "{syscpu:true|false}", - "sysdiskhw", "{sysdiskhw:true|false}", - "sysosinfo", "{sysosinfo:true|false}", - "sysmem", "{sysmem:true|false}", - "sysprocess", "{sysprocess:true|false}", - ) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo"). + HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)) } } diff --git a/cmd/common-main.go b/cmd/common-main.go index d9bac566f..3ad126435 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -43,10 +43,6 @@ func init() { logger.Init(GOPATH, GOROOT) logger.RegisterError(config.FmtError) - // Initialize globalConsoleSys system - globalConsoleSys = NewConsoleLogger(GlobalContext) - logger.AddTarget(globalConsoleSys) - gob.Register(StorageErr("")) } diff --git a/cmd/consolelogger.go b/cmd/consolelogger.go index 7725633ef..cc578223a 100644 --- a/cmd/consolelogger.go +++ b/cmd/consolelogger.go @@ -122,6 +122,24 @@ func (sys *HTTPConsoleLoggerSys) Validate() error { return nil } +// Content returns the console stdout log +func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) { + sys.RLock() + sys.logBuf.Do(func(p interface{}) { + if p != nil { + lg, ok := p.(log.Info) + if ok { + if (lg.Entry != log.Entry{}) { + logs = append(logs, lg.Entry) + } + } + } + }) + sys.RUnlock() + + return +} + // Send log message 'e' to console and publish to console // log pubsub system func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error { diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 2a98aa5af..b116e3694 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -168,6 +168,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) { cli.ShowCommandHelpAndExit(ctx, gatewayName, 1) } + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(GlobalContext) + logger.AddTarget(globalConsoleSys) + // Handle common command args. handleCommonCmdArgs(ctx) diff --git a/cmd/notification.go b/cmd/notification.go index 9b7735179..48f8c17b5 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -1118,6 +1118,36 @@ func (sys *NotificationSys) ProcOBDInfo(ctx context.Context) []madmin.ServerProc return reply } +// LogOBDInfo - Logs OBD information +func (sys *NotificationSys) LogOBDInfo(ctx context.Context) []madmin.ServerLogOBDInfo { + reply := make([]madmin.ServerLogOBDInfo, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].LogOBDInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + addr := sys.peerClients[index].host.String() + reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + logger.LogIf(ctx, err) + reply[index].Addr = addr + reply[index].Error = err.Error() + } + } + return reply +} + // ServerInfo - calls ServerInfo RPC call on all peers. func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties { reply := make([]madmin.ServerProperties, len(sys.peerClients)) diff --git a/cmd/obdinfo.go b/cmd/obdinfo.go index 52fd6aea1..3085228dc 100644 --- a/cmd/obdinfo.go +++ b/cmd/obdinfo.go @@ -148,6 +148,19 @@ func getLocalMemOBD(ctx context.Context, r *http.Request) madmin.ServerMemOBDInf } } +func getLocalLogOBD(ctx context.Context, r *http.Request) madmin.ServerLogOBDInfo { + addr := r.Host + if globalIsDistErasure { + addr = GetLocalPeer(globalEndpoints) + } + + log := globalConsoleSys.Content() + return madmin.ServerLogOBDInfo{ + Addr: addr, + Entries: log, + } +} + func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDInfo { addr := r.Host if globalIsDistErasure { diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go index 4aa597af2..e67daf848 100644 --- a/cmd/peer-rest-client.go +++ b/cmd/peer-rest-client.go @@ -420,6 +420,17 @@ func (client *peerRESTClient) ProcOBDInfo(ctx context.Context) (info madmin.Serv return info, err } +// LogOBDInfo - fetch Log OBD information for a remote node. +func (client *peerRESTClient) LogOBDInfo(ctx context.Context) (info madmin.ServerLogOBDInfo, err error) { + respBody, err := client.callWithContext(ctx, peerRESTMethodLogOBDInfo, nil, nil, -1) + if err != nil { + return + } + defer http.DrainBody(respBody) + err = gob.NewDecoder(respBody).Decode(&info) + return info, err +} + // StartProfiling - Issues profiling command on the peer node. func (client *peerRESTClient) StartProfiling(profiler string) error { values := make(url.Values) diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go index 003ad6257..14c9d288a 100644 --- a/cmd/peer-rest-common.go +++ b/cmd/peer-rest-common.go @@ -33,6 +33,7 @@ const ( peerRESTMethodOsInfoOBDInfo = "/osinfoobdinfo" peerRESTMethodMemOBDInfo = "/memobdinfo" peerRESTMethodProcOBDInfo = "/procobdinfo" + peerRESTMethodLogOBDInfo = "/logobdinfo" peerRESTMethodDispatchNetOBDInfo = "/dispatchnetobdinfo" peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata" peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata" diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 2d8814822..e0db3ce08 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -545,6 +545,22 @@ func (s *peerRESTServer) ProcOBDInfoHandler(w http.ResponseWriter, r *http.Reque logger.LogIf(ctx, gob.NewEncoder(w).Encode(info)) } +// LogOBDInfoHandler - returns Log OBD info. +func (s *peerRESTServer) LogOBDInfoHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + info := getLocalLogOBD(ctx, r) + + defer w.(http.Flusher).Flush() + logger.LogIf(ctx, gob.NewEncoder(w).Encode(info)) +} + // MemOBDInfoHandler - returns Mem OBD info. func (s *peerRESTServer) MemOBDInfoHandler(w http.ResponseWriter, r *http.Request) { if !s.IsValid(w, r) { @@ -1039,6 +1055,7 @@ func registerPeerRESTHandlers(router *mux.Router) { subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLogOBDInfo).HandlerFunc(httpTraceHdrs(server.LogOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemOBDInfo).HandlerFunc(httpTraceHdrs(server.MemOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfoOBDInfo).HandlerFunc(httpTraceHdrs(server.OsOBDInfoHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwOBDInfo).HandlerFunc(httpTraceHdrs(server.DiskHwOBDInfoHandler)) diff --git a/cmd/server-main.go b/cmd/server-main.go index 5e246e6d6..594b5d88e 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -391,6 +391,7 @@ func serverMain(ctx *cli.Context) { // Initialize globalConsoleSys system globalConsoleSys = NewConsoleLogger(GlobalContext) + logger.AddTarget(globalConsoleSys) // Handle all server command args. serverHandleCmdArgs(ctx) diff --git a/pkg/madmin/obd.go b/pkg/madmin/obd.go index c6ebee716..1d53f3d9a 100644 --- a/pkg/madmin/obd.go +++ b/pkg/madmin/obd.go @@ -25,6 +25,7 @@ import ( "net/url" "time" + "github.com/minio/minio/cmd/logger/message/log" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/net" @@ -42,6 +43,19 @@ type OBDInfo struct { Perf PerfOBDInfo `json:"perf,omitempty"` Minio MinioOBDInfo `json:"minio,omitempty"` Sys SysOBDInfo `json:"sys,omitempty"` + Logging LogOBDInfo `json:"logging,omitempty"` +} + +// LogOBDInfo holds different type of logs of all clusters +type LogOBDInfo struct { + ServersLog []ServerLogOBDInfo `json:"serverLogs"` +} + +// ServerLogOBDInfo holds server's stdout log +type ServerLogOBDInfo struct { + Addr string `json:"nodeName"` + Entries []log.Entry `json:"entries"` + Error string `json:"error,omitempty"` } // SysOBDInfo - Includes hardware and system information of the MinIO cluster @@ -185,6 +199,7 @@ const ( OBDDataTypeSysMem OBDDataType = "sysmem" OBDDataTypeSysNet OBDDataType = "sysnet" OBDDataTypeSysProcess OBDDataType = "sysprocess" + OBDDataTypeLog OBDDataType = "log" ) // OBDDataTypesMap - Map of OBD datatypes @@ -201,6 +216,7 @@ var OBDDataTypesMap = map[string]OBDDataType{ "sysmem": OBDDataTypeSysMem, "sysnet": OBDDataTypeSysNet, "sysprocess": OBDDataTypeSysProcess, + "log": OBDDataTypeLog, } // OBDDataTypesList - List of OBD datatypes @@ -217,6 +233,7 @@ var OBDDataTypesList = []OBDDataType{ OBDDataTypeSysMem, OBDDataTypeSysNet, OBDDataTypeSysProcess, + OBDDataTypeLog, } // ServerOBDInfo - Connect to a minio server and call OBD Info Management API