Add admin api to return sys services info (#12939)

The intention is to provide status of any sys services that can
potentially impact the performance of minio.

At present, it will return information about the `selinux` service
(not-installed/disabled/permissive/enforcing)

Signed-off-by: Shireesh Anjal <shireesh@minio.io>

cmd/admin-handlers.go

@@ -1567,6 +1567,22 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	getAndWriteSysServices := func() {
+		if query.Get(string(madmin.HealthDataTypeSysServices)) == "true" {
+			localSysServices := madmin.GetSysServices(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localSysServices)
+			healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, localSysServices)
+			partialWrite(healthInfo)
+
+			peerSysServices := globalNotificationSys.GetSysServices(deadlinedCtx)
+			for _, sli := range peerSysServices {
+				anonymizeAddr(&sli)
+				healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, sli)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
 	anonymizeCmdLine := func(cmdLine string) string {
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
@@ -1741,6 +1757,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		getAndWriteDrivePerfInfo()
 		getAndWriteNetPerfInfo()
 		getAndWriteSysErrors()
+		getAndWriteSysServices()
 
 		if query.Get("minioinfo") == "true" {
 			infoMessage := getServerInfo(ctx, r)

cmd/notification.go

@@ -995,12 +995,7 @@ func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply
@@ -1025,12 +1020,7 @@ func (sys *NotificationSys) GetPartitions(ctx context.Context) []madmin.Partitio
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply
@@ -1055,17 +1045,47 @@ func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply
 }
 
+// GetSysServices - Get information about system services
+// (only the services that are of concern to minio)
+func (sys *NotificationSys) GetSysServices(ctx context.Context) []madmin.SysServices {
+	reply := make([]madmin.SysServices, len(sys.peerClients))
+
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			reply[index], err = sys.peerClients[index].GetSELinuxInfo(ctx)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
+		}
+	}
+	return reply
+}
+
+func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *peerRESTClient, err error) {
+	addr := peerClient.host.String()
+	reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+	logger.LogIf(ctx, err)
+	nodeInfo.SetAddr(addr)
+	nodeInfo.SetError(err.Error())
+}
+
 // GetSysErrors - Memory information
 func (sys *NotificationSys) GetSysErrors(ctx context.Context) []madmin.SysErrors {
 	reply := make([]madmin.SysErrors, len(sys.peerClients))
@@ -1085,12 +1105,7 @@ func (sys *NotificationSys) GetSysErrors(ctx context.Context) []madmin.SysErrors
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply
@@ -1115,12 +1130,7 @@ func (sys *NotificationSys) GetMemInfo(ctx context.Context) []madmin.MemInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply
@@ -1145,12 +1155,7 @@ func (sys *NotificationSys) GetProcInfo(ctx context.Context) []madmin.ProcInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
 		}
 	}
 	return reply

cmd/peer-rest-client.go

@@ -400,6 +400,17 @@ func (client *peerRESTClient) GetOSInfo(ctx context.Context) (info madmin.OSInfo
 	return info, err
 }
 
+// GetSELinuxInfo - fetch SELinux information for a remote node.
+func (client *peerRESTClient) GetSELinuxInfo(ctx context.Context) (info madmin.SysServices, err error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodSysServices, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
 // GetSysErrors - fetch memory information for a remote node.
 func (client *peerRESTClient) GetSysErrors(ctx context.Context) (info madmin.SysErrors, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodSysErrors, nil, nil, -1)

cmd/peer-rest-common.go

@@ -35,6 +35,7 @@ const (
 	peerRESTMethodMemInfo                  = "/meminfo"
 	peerRESTMethodProcInfo                 = "/procinfo"
 	peerRESTMethodSysErrors                = "/syserrors"
+	peerRESTMethodSysServices              = "/sysservices"
 	peerRESTMethodDispatchNetInfo          = "/dispatchnetinfo"
 	peerRESTMethodDeleteBucketMetadata     = "/deletebucketmetadata"
 	peerRESTMethodLoadBucketMetadata       = "/loadbucketmetadata"

cmd/peer-rest-server.go

@@ -510,7 +510,25 @@ func (s *peerRESTServer) GetMemInfoHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
-// GetSysErrorsHandler - returns memory information.
+// GetSysServicesHandler - returns system services information.
+// (only the services that are of concern to minio)
+func (s *peerRESTServer) GetSysServicesHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		fmt.Println("Invalid request")
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	info := madmin.GetSysServices(ctx, r.Host)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// GetSysErrorsHandler - returns system level errors
 func (s *peerRESTServer) GetSysErrorsHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -1301,6 +1319,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(httpTraceHdrs(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.0.20
+	github.com/minio/madmin-go v1.0.21-0.20210812051916-2b99aca7520a
 	github.com/minio/minio-go/v7 v7.0.13-0.20210715203016-9e713532886e
 	github.com/minio/parquet-go v1.0.0
 	github.com/minio/pkg v1.0.11

go.sum

@@ -1023,8 +1023,8 @@ github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/madmin-go v1.0.17/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
-github.com/minio/madmin-go v1.0.20 h1:gb3zY97oUuBuw6BTxmn2THhiEXEgSRmSVqhhHZq3Pc0=
+github.com/minio/madmin-go v1.0.21-0.20210812051916-2b99aca7520a h1:Ha8O6nUq/CjlAc2bYRf7PGdJrikYGwjrSGvRStDcfWU=
-github.com/minio/madmin-go v1.0.20/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
+github.com/minio/madmin-go v1.0.21-0.20210812051916-2b99aca7520a/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f h1:hyFvo5hSFw2K417YvDr/vAKlgCG69uTuhZW/5LNdL0U=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f/go.mod h1:tuaonkPjVApCXkbtKENHBtsqUf7YTV33qmFrC+Pgp5g=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=