Add ServerCPULoadInfo() and ServerMemUsageInfo() admin API (#7038)

pkg/cpu/counter_darwin.go

@@ -0,0 +1,29 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"errors"
+)
+
+func newCounter() (counter, error) {
+	return counter{}, errors.New("cpu metrics not implemented for darwin platform")
+}
+
+func (c counter) now() time.Time {
+	return time.Time{}
+}

pkg/cpu/counter_linux.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+const (
+	// ProcessClock corresponds to the High-resolution per-process
+	// timer from the CPU represented in stdlib as CLOCK_PROCESS_CPUTIME_ID
+	processClock = 2
+)
+
+func newCounter() (counter, error) {
+	return counter{}, nil
+}
+
+func (c counter) now() time.Time {
+	var ts syscall.Timespec
+	syscall.Syscall(syscall.SYS_CLOCK_GETTIME, processClock, uintptr(unsafe.Pointer(&ts)), 0)
+	sec, nsec := ts.Unix()
+	return time.Unix(sec, nsec)
+}

pkg/cpu/counter_windows.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"errors"
+	"time"
+)
+
+func newCounter() (counter, error) {
+	return counter{}, errors.New("cpu metrics not implemented for windows platform")
+}
+
+func (c counter) now() time.Time {
+	return time.Time{}
+}

pkg/cpu/cpu.go

@@ -0,0 +1,91 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+const (
+	// cpuLoadWindow is the interval of time for which the
+	// cpu utilization is measured
+	cpuLoadWindow = 200 * time.Millisecond
+
+	// cpuLoadSampleSize is the number of samples measured
+	// for calculating cpu utilization
+	cpuLoadSampleSize = 3
+
+	// endOfTime represents the end of time
+	endOfTime = time.Duration(1<<63 - 1)
+)
+
+// Load holds CPU utilization % measured in three intervals of 200ms each
+type Load struct {
+	Avg   string `json:"avg"`
+	Max   string `json:"max"`
+	Min   string `json:"min"`
+	Error string `json:"error,omitempty"`
+}
+
+type counter struct{}
+
+// GetLoad returns the CPU utilization % of the current process
+func GetLoad() Load {
+	vals := make(chan time.Duration, 3)
+	wg := sync.WaitGroup{}
+	for i := 0; i < cpuLoadSampleSize; i++ {
+		cpuCounter, err := newCounter()
+		if err != nil {
+			return Load{
+				Error: err.Error(),
+			}
+		}
+		wg.Add(1)
+		go func() {
+			start := cpuCounter.now()
+			time.Sleep(cpuLoadWindow)
+			end := cpuCounter.now()
+			vals <- end.Sub(start)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	sum := time.Duration(0)
+	max := time.Duration(0)
+	min := (endOfTime)
+	for i := 0; i < cpuLoadSampleSize; i++ {
+		val := <-vals
+		sum = sum + val
+		if val > max {
+			max = val
+		}
+		if val < min {
+			min = val
+		}
+	}
+	close(vals)
+	avg := sum / 3
+	return Load{
+		Avg:   fmt.Sprintf("%.2f%%", toFixed4(float64(avg)/float64(200*time.Millisecond))*100),
+		Max:   fmt.Sprintf("%.2f%%", toFixed4(float64(max)/float64(200*time.Millisecond))*100),
+		Min:   fmt.Sprintf("%.2f%%", toFixed4(float64(min)/float64(200*time.Millisecond))*100),
+		Error: "",
+	}
+}

pkg/cpu/helpers.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"math"
+)
+
+func round(num float64) int {
+	return int(num + math.Copysign(0.5, num))
+}
+
+func toFixed4(num float64) float64 {
+	output := math.Pow(10, float64(4))
+	return float64(round(num*output)) / output
+}

pkg/madmin/API.md

@@ -39,7 +39,7 @@ func main() {
 | Service operations         | Info operations  | Healing operations                    | Config operations        | IAM operations | Misc                                |
 |:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------|
 | [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
+| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | [`ServerMemUsageInfo`](#ServerMemUsageInfo)  | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
 | | |            | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
 | | |            | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | |
 
@@ -222,6 +222,40 @@ Fetches drive performance information for all cluster nodes. Returned value is i
 |`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. |
 |`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. |
 
+<a name="ServerCPULoadInfo"></a>
+### ServerCPULoadInfo() ([]ServerCPULoadInfo, error)
+
+Fetches CPU utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`cpui.Addr` | _string_ | Address of the server the following information  is retrieved from. |
+|`cpui.Error` | _string_ | Errors (if any) encountered while reaching this node |
+|`cpui.CPULoad` | _cpu.Load_ | The load on the CPU. |
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`cpu.Load.Avg` | _string_ | The average utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Min` | _string_ | The minimum utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Max` | _string_ | The maximum utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
+
+<a name="ServerMemUsageInfo"></a>
+### ServerMemUsageInfo() ([]ServerMemUsageInfo, error)
+
+Fetches Mem utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`memi.Addr` | _string_ | Address of the server the following information  is retrieved from. |
+|`memi.Error` | _string_ | Errors (if any) encountered while reaching this node |
+|`memi.MemUsage` | _mem.Usage_ | The utilitzation of Memory |
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`mem.Usage.Mem` | _string_ | The total number of bytes obtained from the OS |
+|`mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
+
 ## 6. Heal operations
 
 <a name="Heal"></a>

pkg/madmin/examples/cpu-load-info.go

@@ -0,0 +1,44 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	st, err := madmClnt.ServerCPULoadInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(st)
+}

pkg/madmin/examples/mem-usage-info.go

@@ -0,0 +1,44 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	st, err := madmClnt.ServerMemUsageInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(st)
+}

pkg/madmin/info-commands.go

@@ -24,7 +24,9 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mem"
 )
 
 // BackendType - represents different backend types.
@@ -193,3 +195,89 @@ func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
 
 	return info, nil
 }
+
+// ServerCPULoadInfo holds information about address and cpu load of
+// a single server node
+type ServerCPULoadInfo struct {
+	Addr  string     `json:"addr"`
+	Error string     `json:"error,omitempty"`
+	Load  []cpu.Load `json:"load"`
+}
+
+// ServerCPULoadInfo - Returns cpu utilization information
+func (adm *AdminClient) ServerCPULoadInfo() ([]ServerCPULoadInfo, error) {
+	v := url.Values{}
+	v.Set("perfType", string("cpu"))
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath:     "/v1/performance",
+		queryValues: v,
+	})
+
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var info []ServerCPULoadInfo
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// ServerMemUsageInfo holds information about address and memory utilization of
+// a single server node
+type ServerMemUsageInfo struct {
+	Addr  string      `json:"addr"`
+	Error string      `json:"error,omitempty"`
+	Usage []mem.Usage `json:"usage"`
+}
+
+// ServerMemUsageInfo - Returns mem utilization information
+func (adm *AdminClient) ServerMemUsageInfo() ([]ServerMemUsageInfo, error) {
+	v := url.Values{}
+	v.Set("perfType", string("mem"))
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath:     "/v1/performance",
+		queryValues: v,
+	})
+
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var info []ServerMemUsageInfo
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}

pkg/mem/mem.go

@@ -0,0 +1,39 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package mem
+
+import (
+	"runtime"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+// Usage holds memory utilization information in human readable format
+type Usage struct {
+	Mem   string `json:"mem"`
+	Error string `json:"error,omitempty"`
+}
+
+// GetUsage measures the total memory provisioned for the current process
+// from the OS
+func GetUsage() Usage {
+	memStats := new(runtime.MemStats)
+	runtime.ReadMemStats(memStats)
+	return Usage{
+		Mem: humanize.IBytes(memStats.Sys),
+	}
+}