/* * MinIO Cloud Storage, (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package madmin import ( "context" "encoding/json" "io" "net/http" "net/url" "time" "github.com/minio/minio/cmd/logger/message/log" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/net" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/mem" nethw "github.com/shirou/gopsutil/net" "github.com/shirou/gopsutil/process" ) // OBDInfo - MinIO cluster's OBD Info type OBDInfo struct { TimeStamp time.Time `json:"timestamp,omitempty"` Error string `json:"error,omitempty"` Perf PerfOBDInfo `json:"perf,omitempty"` Minio MinioOBDInfo `json:"minio,omitempty"` Sys SysOBDInfo `json:"sys,omitempty"` Logging LogOBDInfo `json:"logging,omitempty"` } // LogOBDInfo holds different type of logs of all clusters type LogOBDInfo struct { ServersLog []ServerLogOBDInfo `json:"serverLogs"` } // ServerLogOBDInfo holds server's stdout log type ServerLogOBDInfo struct { Addr string `json:"nodeName"` Entries []log.Entry `json:"entries"` Error string `json:"error,omitempty"` } // SysOBDInfo - Includes hardware and system information of the MinIO cluster type SysOBDInfo struct { CPUInfo []ServerCPUOBDInfo `json:"cpus,omitempty"` DiskHwInfo []ServerDiskHwOBDInfo `json:"disks,omitempty"` OsInfo []ServerOsOBDInfo `json:"osinfos,omitempty"` MemInfo []ServerMemOBDInfo `json:"meminfos,omitempty"` ProcInfo []ServerProcOBDInfo `json:"procinfos,omitempty"` Error string `json:"error,omitempty"` } // ServerProcOBDInfo - Includes host process lvl information type ServerProcOBDInfo struct { Addr string `json:"addr"` Processes []SysOBDProcess `json:"processes,omitempty"` Error string `json:"error,omitempty"` } // SysOBDProcess - Includes process lvl information about a single process type SysOBDProcess struct { Pid int32 `json:"pid"` Background bool `json:"background,omitempty"` CPUPercent float64 `json:"cpupercent,omitempty"` Children []int32 `json:"children,omitempty"` CmdLine string `json:"cmd,omitempty"` Connections []nethw.ConnectionStat `json:"connections,omitempty"` CreateTime int64 `json:"createtime,omitempty"` Cwd string `json:"cwd,omitempty"` Exe string `json:"exe,omitempty"` Gids []int32 `json:"gids,omitempty"` IOCounters *process.IOCountersStat `json:"iocounters,omitempty"` IsRunning bool `json:"isrunning,omitempty"` MemInfo *process.MemoryInfoStat `json:"meminfo,omitempty"` MemMaps *[]process.MemoryMapsStat `json:"memmaps,omitempty"` MemPercent float32 `json:"mempercent,omitempty"` Name string `json:"name,omitempty"` NetIOCounters []nethw.IOCountersStat `json:"netiocounters,omitempty"` Nice int32 `json:"nice,omitempty"` NumCtxSwitches *process.NumCtxSwitchesStat `json:"numctxswitches,omitempty"` NumFds int32 `json:"numfds,omitempty"` NumThreads int32 `json:"numthreads,omitempty"` OpenFiles []process.OpenFilesStat `json:"openfiles,omitempty"` PageFaults *process.PageFaultsStat `json:"pagefaults,omitempty"` Parent int32 `json:"parent,omitempty"` Ppid int32 `json:"ppid,omitempty"` Rlimit []process.RlimitStat `json:"rlimit,omitempty"` Status string `json:"status,omitempty"` Tgid int32 `json:"tgid,omitempty"` Threads map[int32]*cpu.TimesStat `json:"threadstats,omitempty"` Times *cpu.TimesStat `json:"cputimes,omitempty"` Uids []int32 `json:"uidsomitempty"` Username string `json:"username,omitempty"` } // ServerMemOBDInfo - Includes host virtual and swap mem information type ServerMemOBDInfo struct { Addr string `json:"addr"` SwapMem *mem.SwapMemoryStat `json:"swap,omitempty"` VirtualMem *mem.VirtualMemoryStat `json:"virtualmem,omitempty"` Error string `json:"error,omitempty"` } // ServerOsOBDInfo - Includes host os information type ServerOsOBDInfo struct { Addr string `json:"addr"` Info *host.InfoStat `json:"info,omitempty"` Sensors []host.TemperatureStat `json:"sensors,omitempty"` Users []host.UserStat `json:"users,omitempty"` Error string `json:"error,omitempty"` } // ServerCPUOBDInfo - Includes cpu and timer stats of each node of the MinIO cluster type ServerCPUOBDInfo struct { Addr string `json:"addr"` CPUStat []cpu.InfoStat `json:"cpu,omitempty"` TimeStat []cpu.TimesStat `json:"time,omitempty"` Error string `json:"error,omitempty"` } // MinioOBDInfo - Includes MinIO confifuration information type MinioOBDInfo struct { Info InfoMessage `json:"info,omitempty"` Config interface{} `json:"config,omitempty"` Error string `json:"error,omitempty"` } // PerfOBDInfo - Includes Drive and Net perf info for the entire MinIO cluster type PerfOBDInfo struct { DriveInfo []ServerDrivesOBDInfo `json:"drives,omitempty"` Net []ServerNetOBDInfo `json:"net,omitempty"` NetParallel ServerNetOBDInfo `json:"net_parallel,omitempty"` Error string `json:"error,omitempty"` } // ServerDrivesOBDInfo - Drive OBD info about all drives in a single MinIO node type ServerDrivesOBDInfo struct { Addr string `json:"addr"` Serial []DriveOBDInfo `json:"serial,omitempty"` Parallel []DriveOBDInfo `json:"parallel,omitempty"` Error string `json:"error,omitempty"` } // DriveOBDInfo - Stats about a single drive in a MinIO node type DriveOBDInfo struct { Path string `json:"endpoint"` Latency disk.Latency `json:"latency,omitempty"` Throughput disk.Throughput `json:"throughput,omitempty"` Error string `json:"error,omitempty"` } // ServerNetOBDInfo - Network OBD info about a single MinIO node type ServerNetOBDInfo struct { Addr string `json:"addr"` Net []NetOBDInfo `json:"net,omitempty"` Error string `json:"error,omitempty"` } // NetOBDInfo - one-to-one network connectivity Stats between 2 MinIO nodes type NetOBDInfo struct { Addr string `json:"remote"` Latency net.Latency `json:"latency,omitempty"` Throughput net.Throughput `json:"throughput,omitempty"` Error string `json:"error,omitempty"` } // OBDDataType - Typed OBD data types type OBDDataType string // OBDDataTypes const ( OBDDataTypePerfDrive OBDDataType = "perfdrive" OBDDataTypePerfNet OBDDataType = "perfnet" OBDDataTypeMinioInfo OBDDataType = "minioinfo" OBDDataTypeMinioConfig OBDDataType = "minioconfig" OBDDataTypeSysCPU OBDDataType = "syscpu" OBDDataTypeSysDiskHw OBDDataType = "sysdiskhw" OBDDataTypeSysDocker OBDDataType = "sysdocker" // is this really needed? OBDDataTypeSysOsInfo OBDDataType = "sysosinfo" OBDDataTypeSysLoad OBDDataType = "sysload" // provides very little info. Making it TBD OBDDataTypeSysMem OBDDataType = "sysmem" OBDDataTypeSysNet OBDDataType = "sysnet" OBDDataTypeSysProcess OBDDataType = "sysprocess" OBDDataTypeLog OBDDataType = "log" ) // OBDDataTypesMap - Map of OBD datatypes var OBDDataTypesMap = map[string]OBDDataType{ "perfdrive": OBDDataTypePerfDrive, "perfnet": OBDDataTypePerfNet, "minioinfo": OBDDataTypeMinioInfo, "minioconfig": OBDDataTypeMinioConfig, "syscpu": OBDDataTypeSysCPU, "sysdiskhw": OBDDataTypeSysDiskHw, "sysdocker": OBDDataTypeSysDocker, "sysosinfo": OBDDataTypeSysOsInfo, "sysload": OBDDataTypeSysLoad, "sysmem": OBDDataTypeSysMem, "sysnet": OBDDataTypeSysNet, "sysprocess": OBDDataTypeSysProcess, "log": OBDDataTypeLog, } // OBDDataTypesList - List of OBD datatypes var OBDDataTypesList = []OBDDataType{ OBDDataTypePerfDrive, OBDDataTypePerfNet, OBDDataTypeMinioInfo, OBDDataTypeMinioConfig, OBDDataTypeSysCPU, OBDDataTypeSysDiskHw, OBDDataTypeSysDocker, OBDDataTypeSysOsInfo, OBDDataTypeSysLoad, OBDDataTypeSysMem, OBDDataTypeSysNet, OBDDataTypeSysProcess, OBDDataTypeLog, } // ServerOBDInfo - Connect to a minio server and call OBD Info Management API // to fetch server's information represented by OBDInfo structure func (adm *AdminClient) ServerOBDInfo(ctx context.Context, obdDataTypes []OBDDataType, deadline time.Duration) <-chan OBDInfo { respChan := make(chan OBDInfo) go func() { v := url.Values{} v.Set("deadline", deadline.Truncate(1*time.Second).String()) // start with all set to false for _, d := range OBDDataTypesList { v.Set(string(d), "false") } // only 'trueify' user provided values for _, d := range obdDataTypes { v.Set(string(d), "true") } var OBDInfoMessage OBDInfo OBDInfoMessage.TimeStamp = time.Now() if v.Get(string(OBDDataTypeMinioInfo)) == "true" { info, err := adm.ServerInfo(ctx) if err != nil { respChan <- OBDInfo{ Error: err.Error(), } return } OBDInfoMessage.Minio.Info = info respChan <- OBDInfoMessage } resp, err := adm.executeMethod(ctx, "GET", requestData{ relPath: adminAPIPrefix + "/obdinfo", queryValues: v, }) defer closeResponse(resp) if err != nil { respChan <- OBDInfo{ Error: err.Error(), } close(respChan) return } // Check response http status code if resp.StatusCode != http.StatusOK { respChan <- OBDInfo{ Error: httpRespToErrorResponse(resp).Error(), } return } // Unmarshal the server's json response decoder := json.NewDecoder(resp.Body) for { err := decoder.Decode(&OBDInfoMessage) OBDInfoMessage.TimeStamp = time.Now() if err == io.EOF { break } if err != nil { respChan <- OBDInfo{ Error: err.Error(), } } respChan <- OBDInfoMessage } respChan <- OBDInfoMessage close(respChan) }() return respChan }