From 30c25965120a3043a6e5d65f88ff917b0f43ff9f Mon Sep 17 00:00:00 2001 From: Praveen raj Mani Date: Tue, 27 Feb 2024 01:04:50 +0530 Subject: [PATCH] Read drive IO stats from sysfs instead of procfs (#19131) Currently, we read from `/proc/diskstats` which is found to be un-reliable in k8s environments. We can read from `sysfs` instead. Also, cache the latest drive io stats to find the diff and update the metrics. --- cmd/metrics-realtime.go | 11 +- cmd/metrics-resource.go | 71 ++++++------- internal/disk/disk.go | 9 -- internal/disk/stat_bsd.go | 6 +- internal/disk/stat_freebsd.go | 6 +- internal/disk/stat_linux.go | 166 ++++++++++-------------------- internal/disk/stat_linux_32bit.go | 6 +- internal/disk/stat_linux_s390x.go | 6 +- internal/disk/stat_netbsd.go | 6 +- internal/disk/stat_openbsd.go | 6 +- internal/disk/stat_solaris.go | 6 +- internal/disk/stat_test.go | 130 +++++++++++++++++++++++ internal/disk/stat_windows.go | 6 +- 13 files changed, 243 insertions(+), 192 deletions(-) create mode 100644 internal/disk/stat_test.go diff --git a/cmd/metrics-realtime.go b/cmd/metrics-realtime.go index 83fe918af..d704969ca 100644 --- a/cmd/metrics-realtime.go +++ b/cmd/metrics-realtime.go @@ -130,12 +130,6 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM } metrics := make(map[string]madmin.DiskMetric) - - procStats, procErr := disk.GetAllDrivesIOStats() - if procErr != nil { - return metrics - } - storageInfo := objLayer.LocalStorageInfo(GlobalContext, true) for _, d := range storageInfo.Disks { if len(disks) != 0 { @@ -170,9 +164,8 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM } } - // get disk - if procErr == nil { - st := procStats[disk.DevID{Major: d.Major, Minor: d.Minor}] + st, err := disk.GetDriveStats(d.Major, d.Minor) + if err == nil { dm.IOStats = madmin.DiskIOStats{ ReadIOs: st.ReadIOs, ReadMerges: st.ReadMerges, diff --git a/cmd/metrics-resource.go b/cmd/metrics-resource.go index 0734d811b..6d052fefb 100644 --- a/cmd/metrics-resource.go +++ b/cmd/metrics-resource.go @@ -27,7 +27,6 @@ import ( "github.com/minio/madmin-go/v3" "github.com/prometheus/client_golang/prometheus" - "github.com/shirou/gopsutil/v3/host" ) const ( @@ -85,9 +84,9 @@ var ( resourceMetricsGroups []*MetricsGroup // initial values for drives (at the time of server startup) // used for calculating avg values for drive metrics - initialDriveStats map[string]madmin.DiskIOStats - initialDriveStatsMu sync.RWMutex - initialUptime uint64 + latestDriveStats map[string]madmin.DiskIOStats + latestDriveStatsMu sync.RWMutex + lastDriveStatsRefresh time.Time ) // PeerResourceMetrics represents the resource metrics @@ -147,7 +146,7 @@ func init() { writesKBPerSec: "Kilobytes written per second on a drive", readsAwait: "Average time for read requests to be served on a drive", writesAwait: "Average time for write requests to be served on a drive", - percUtil: "Percentage of time the disk was busy since uptime", + percUtil: "Percentage of time the disk was busy", usedBytes: "Used bytes on a drive", totalBytes: "Total bytes on a drive", usedInodes: "Total inodes used on a drive", @@ -219,35 +218,32 @@ func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, resourceMetricsMap[subSys] = subsysMetrics } -// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current -// and initial values. We cannot rely on host.Uptime here as it will not work in k8s environments, where -// it will return the pod's uptime but the disk metrics are always from the host (/proc/diskstats) -func updateDriveIOStats(currentStats madmin.DiskIOStats, initialStats madmin.DiskIOStats, labels map[string]string) { +// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current and latest updated values. +func updateDriveIOStats(currentStats madmin.DiskIOStats, latestStats madmin.DiskIOStats, labels map[string]string) { sectorSize := uint64(512) kib := float64(1 << 10) - uptime, _ := host.Uptime() - uptimeDiff := float64(uptime - initialUptime) - if uptimeDiff == 0 { + diffInSeconds := time.Now().UTC().Sub(lastDriveStatsRefresh).Seconds() + if diffInSeconds == 0 { // too soon to update the stats return } diffStats := madmin.DiskIOStats{ - ReadIOs: currentStats.ReadIOs - initialStats.ReadIOs, - WriteIOs: currentStats.WriteIOs - initialStats.WriteIOs, - ReadTicks: currentStats.ReadTicks - initialStats.ReadTicks, - WriteTicks: currentStats.WriteTicks - initialStats.WriteTicks, - TotalTicks: currentStats.TotalTicks - initialStats.TotalTicks, - ReadSectors: currentStats.ReadSectors - initialStats.ReadSectors, - WriteSectors: currentStats.WriteSectors - initialStats.WriteSectors, + ReadIOs: currentStats.ReadIOs - latestStats.ReadIOs, + WriteIOs: currentStats.WriteIOs - latestStats.WriteIOs, + ReadTicks: currentStats.ReadTicks - latestStats.ReadTicks, + WriteTicks: currentStats.WriteTicks - latestStats.WriteTicks, + TotalTicks: currentStats.TotalTicks - latestStats.TotalTicks, + ReadSectors: currentStats.ReadSectors - latestStats.ReadSectors, + WriteSectors: currentStats.WriteSectors - latestStats.WriteSectors, } - updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/uptimeDiff, labels, false) + updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/diffInSeconds, labels, false) readKib := float64(diffStats.ReadSectors*sectorSize) / kib - updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/uptimeDiff, labels, false) + updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/diffInSeconds, labels, false) - updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/uptimeDiff, labels, false) + updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/diffInSeconds, labels, false) writeKib := float64(diffStats.WriteSectors*sectorSize) / kib - updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/uptimeDiff, labels, false) + updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/diffInSeconds, labels, false) rdAwait := 0.0 if diffStats.ReadIOs > 0 { @@ -260,18 +256,23 @@ func updateDriveIOStats(currentStats madmin.DiskIOStats, initialStats madmin.Dis wrAwait = float64(diffStats.WriteTicks) / float64(diffStats.WriteIOs) } updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false) - updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(uptimeDiff*10), labels, false) + updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(diffInSeconds*10), labels, false) } func collectDriveMetrics(m madmin.RealtimeMetrics) { + latestDriveStatsMu.Lock() for d, dm := range m.ByDisk { labels := map[string]string{"drive": d} - initialStats, ok := initialDriveStats[d] + latestStats, ok := latestDriveStats[d] if !ok { + latestDriveStats[d] = dm.IOStats continue } - updateDriveIOStats(dm.IOStats, initialStats, labels) + updateDriveIOStats(dm.IOStats, latestStats, labels) + latestDriveStats[d] = dm.IOStats } + lastDriveStatsRefresh = time.Now().UTC() + latestDriveStatsMu.Unlock() globalLocalDrivesMu.RLock() localDrives := cloneDrives(globalLocalDrives) @@ -361,29 +362,25 @@ func collectLocalResourceMetrics() { collectDriveMetrics(m) } -// populateInitialValues - populates the initial values -// for drive stats and host uptime -func populateInitialValues() { - initialDriveStatsMu.Lock() - +func initLatestValues() { m := collectLocalMetrics(madmin.MetricsDisk, collectMetricsOpts{ hosts: map[string]struct{}{ globalLocalNodeName: {}, }, }) - initialDriveStats = map[string]madmin.DiskIOStats{} + latestDriveStatsMu.Lock() + latestDriveStats = map[string]madmin.DiskIOStats{} for d, dm := range m.ByDisk { - initialDriveStats[d] = dm.IOStats + latestDriveStats[d] = dm.IOStats } - - initialUptime, _ = host.Uptime() - initialDriveStatsMu.Unlock() + lastDriveStatsRefresh = time.Now().UTC() + latestDriveStatsMu.Unlock() } // startResourceMetricsCollection - starts the job for collecting resource metrics func startResourceMetricsCollection() { - populateInitialValues() + initLatestValues() resourceMetricsMapMu.Lock() resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{} diff --git a/internal/disk/disk.go b/internal/disk/disk.go index 3b891edf3..a0b51f123 100644 --- a/internal/disk/disk.go +++ b/internal/disk/disk.go @@ -40,15 +40,6 @@ type Info struct { NRRequests uint64 } -// DevID is the drive major and minor ids -type DevID struct { - Major uint32 - Minor uint32 -} - -// AllDrivesIOStats is map between drive devices and IO stats -type AllDrivesIOStats map[DevID]IOStats - // IOStats contains stats of a single drive type IOStats struct { ReadIOs uint64 diff --git a/internal/disk/stat_bsd.go b/internal/disk/stat_bsd.go index 248f7d5b7..b0d8e7411 100644 --- a/internal/disk/stat_bsd.go +++ b/internal/disk/stat_bsd.go @@ -48,7 +48,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_freebsd.go b/internal/disk/stat_freebsd.go index 04db5d4ab..f76c9cf24 100644 --- a/internal/disk/stat_freebsd.go +++ b/internal/disk/stat_freebsd.go @@ -48,7 +48,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_linux.go b/internal/disk/stat_linux.go index 0409f359b..33f757022 100644 --- a/internal/disk/stat_linux.go +++ b/internal/disk/stat_linux.go @@ -22,7 +22,9 @@ package disk import ( "bufio" + "errors" "fmt" + "io" "os" "path/filepath" "strconv" @@ -109,125 +111,63 @@ func GetInfo(path string, firstTime bool) (info Info, err error) { return info, nil } -const ( - statsPath = "/proc/diskstats" -) +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return readDriveStats(fmt.Sprintf("/sys/dev/block/%v:%v/stat", major, minor)) +} -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - proc, err := os.Open(statsPath) +func readDriveStats(statsFile string) (iostats IOStats, err error) { + stats, err := readStat(statsFile) + if err != nil { + return IOStats{}, err + } + if len(stats) < 11 { + return IOStats{}, fmt.Errorf("found invalid format while reading %v", statsFile) + } + // refer https://www.kernel.org/doc/Documentation/block/stat.txt + iostats = IOStats{ + ReadIOs: stats[0], + ReadMerges: stats[1], + ReadSectors: stats[2], + ReadTicks: stats[3], + WriteIOs: stats[4], + WriteMerges: stats[5], + WriteSectors: stats[6], + WriteTicks: stats[7], + CurrentIOs: stats[8], + TotalTicks: stats[9], + ReqTicks: stats[10], + } + // as per the doc, only 11 fields are guaranteed + // only set if available + if len(stats) > 14 { + iostats.DiscardIOs = stats[11] + iostats.DiscardMerges = stats[12] + iostats.DiscardSectors = stats[13] + iostats.DiscardTicks = stats[14] + } + return +} + +func readStat(fileName string) (stats []uint64, err error) { + file, err := os.Open(fileName) if err != nil { return nil, err } - defer proc.Close() + defer file.Close() - ret := make(AllDrivesIOStats) - - sc := bufio.NewScanner(proc) - for sc.Scan() { - line := sc.Text() - fields := strings.Fields(line) - if len(fields) < 11 { - continue - } - - var err error - var ds IOStats - - ds.ReadIOs, err = strconv.ParseUint((fields[3]), 10, 64) - if err != nil { - return ret, err - } - ds.ReadMerges, err = strconv.ParseUint((fields[4]), 10, 64) - if err != nil { - return ret, err - } - ds.ReadSectors, err = strconv.ParseUint((fields[5]), 10, 64) - if err != nil { - return ret, err - } - ds.ReadTicks, err = strconv.ParseUint((fields[6]), 10, 64) - if err != nil { - return ret, err - } - ds.WriteIOs, err = strconv.ParseUint((fields[7]), 10, 64) - if err != nil { - return ret, err - } - ds.WriteMerges, err = strconv.ParseUint((fields[8]), 10, 64) - if err != nil { - return ret, err - } - ds.WriteSectors, err = strconv.ParseUint((fields[9]), 10, 64) - if err != nil { - return ret, err - } - ds.WriteTicks, err = strconv.ParseUint((fields[10]), 10, 64) - if err != nil { - return ret, err - } - - if len(fields) > 11 { - ds.CurrentIOs, err = strconv.ParseUint((fields[11]), 10, 64) - if err != nil { - return ret, err - } - - ds.TotalTicks, err = strconv.ParseUint((fields[12]), 10, 64) - if err != nil { - return ret, err - } - ds.ReqTicks, err = strconv.ParseUint((fields[13]), 10, 64) - if err != nil { - return ret, err - } - } - - if len(fields) > 14 { - ds.DiscardIOs, err = strconv.ParseUint((fields[14]), 10, 64) - if err != nil { - return ret, err - } - ds.DiscardMerges, err = strconv.ParseUint((fields[15]), 10, 64) - if err != nil { - return ret, err - } - ds.DiscardSectors, err = strconv.ParseUint((fields[16]), 10, 64) - if err != nil { - return ret, err - } - ds.DiscardTicks, err = strconv.ParseUint((fields[17]), 10, 64) - if err != nil { - return ret, err - } - } - - if len(fields) > 18 { - ds.FlushIOs, err = strconv.ParseUint((fields[18]), 10, 64) - if err != nil { - return ret, err - } - ds.FlushTicks, err = strconv.ParseUint((fields[19]), 10, 64) - if err != nil { - return ret, err - } - } - - major, err := strconv.ParseUint((fields[0]), 10, 32) - if err != nil { - return ret, err - } - - minor, err := strconv.ParseUint((fields[1]), 10, 32) - if err != nil { - return ret, err - } - ret[DevID{uint32(major), uint32(minor)}] = ds - } - - if err := sc.Err(); err != nil { + s, err := bufio.NewReader(file).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { return nil, err } + statLine := strings.TrimSpace(s) + for _, token := range strings.Fields(statLine) { + ui64, err := strconv.ParseUint(token, 10, 64) + if err != nil { + return nil, err + } + stats = append(stats, ui64) + } - return ret, nil + return stats, nil } diff --git a/internal/disk/stat_linux_32bit.go b/internal/disk/stat_linux_32bit.go index a6a53d8bc..ee4384083 100644 --- a/internal/disk/stat_linux_32bit.go +++ b/internal/disk/stat_linux_32bit.go @@ -83,7 +83,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_linux_s390x.go b/internal/disk/stat_linux_s390x.go index e4b323841..5ee6eabdd 100644 --- a/internal/disk/stat_linux_s390x.go +++ b/internal/disk/stat_linux_s390x.go @@ -83,7 +83,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_netbsd.go b/internal/disk/stat_netbsd.go index 490f572ee..ba368bd4f 100644 --- a/internal/disk/stat_netbsd.go +++ b/internal/disk/stat_netbsd.go @@ -48,7 +48,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_openbsd.go b/internal/disk/stat_openbsd.go index 213534531..f1ec7dd68 100644 --- a/internal/disk/stat_openbsd.go +++ b/internal/disk/stat_openbsd.go @@ -48,7 +48,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_solaris.go b/internal/disk/stat_solaris.go index 36c1ec8d0..0409c0ecb 100644 --- a/internal/disk/stat_solaris.go +++ b/internal/disk/stat_solaris.go @@ -48,7 +48,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") } diff --git a/internal/disk/stat_test.go b/internal/disk/stat_test.go new file mode 100644 index 000000000..a33d2e68f --- /dev/null +++ b/internal/disk/stat_test.go @@ -0,0 +1,130 @@ +//go:build linux && !s390x && !arm && !386 +// +build linux,!s390x,!arm,!386 + +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + "reflect" + "runtime" + "testing" +) + +func TestReadDriveStats(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping this test in windows") + } + testCases := []struct { + stat string + expectedIOStats IOStats + expectErr bool + }{ + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354 46037 0 41695120 1315 0 0", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + DiscardIOs: 46037, + DiscardMerges: 0, + DiscardSectors: 41695120, + DiscardTicks: 1315, + FlushIOs: 0, + FlushTicks: 0, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354 46037 0 41695120 1315", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + DiscardIOs: 46037, + DiscardMerges: 0, + DiscardSectors: 41695120, + DiscardTicks: 1315, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227", + expectedIOStats: IOStats{}, + expectErr: true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + tmpfile, err := os.CreateTemp("", "testfile") + if err != nil { + t.Error(err) + } + tmpfile.WriteString(testCase.stat) + tmpfile.Sync() + tmpfile.Close() + + iostats, err := readDriveStats(tmpfile.Name()) + if err != nil && !testCase.expectErr { + t.Fatalf("unexpected err; %v", err) + } + if testCase.expectErr && err == nil { + t.Fatal("expected to fail but err is nil") + } + if !reflect.DeepEqual(iostats, testCase.expectedIOStats) { + t.Fatalf("expected iostats: %v but got %v", testCase.expectedIOStats, iostats) + } + }) + } +} diff --git a/internal/disk/stat_windows.go b/internal/disk/stat_windows.go index f1236a660..7037932b1 100644 --- a/internal/disk/stat_windows.go +++ b/internal/disk/stat_windows.go @@ -108,7 +108,7 @@ func GetInfo(path string, _ bool) (info Info, err error) { return info, nil } -// GetAllDrivesIOStats returns IO stats of all drives found in the machine -func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) { - return nil, errors.New("operation unsupported") +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") }