minio/cmd/os-instrumented.go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"os"
	"strings"
	"sync/atomic"
	"time"

	"github.com/minio/madmin-go/v3"
	"github.com/minio/minio/internal/disk"
	ioutilx "github.com/minio/minio/internal/ioutil"
)

//go:generate stringer -type=osMetric -trimprefix=osMetric $GOFILE

type osMetric uint8

const (
	osMetricRemoveAll osMetric = iota
	osMetricMkdirAll
	osMetricMkdir
	osMetricRename
	osMetricOpenFileW
	osMetricOpenFileR
	osMetricOpen
	osMetricOpenFileDirectIO
	osMetricLstat
	osMetricRemove
	osMetricStat
	osMetricAccess
	osMetricCreate
	osMetricReadDirent
	osMetricFdatasync
	osMetricSync
	osMetricRename2 // Linux specific
	// .... add more

	osMetricLast
)

var globalOSMetrics osMetrics

func init() {
	// Inject metrics.
	ioutilx.OsOpenFile = OpenFile
	ioutilx.OpenFileDirectIO = OpenFileDirectIO
	ioutilx.OsOpen = Open
}

type osMetrics struct {
	// All fields must be accessed atomically and aligned.
	operations [osMetricLast]uint64
	latency    [osMetricLast]lockedLastMinuteLatency
}

// time an os action.
func (o *osMetrics) time(s osMetric) func() {
	startTime := time.Now()
	return func() {
		duration := time.Since(startTime)

		atomic.AddUint64(&o.operations[s], 1)
		o.latency[s].add(duration)
	}
}

// incTime will increment time on metric s with a specific duration.
func (o *osMetrics) incTime(s osMetric, d time.Duration) {
	atomic.AddUint64(&o.operations[s], 1)
	o.latency[s].add(d)
}

func osTrace(s osMetric, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
	var errStr string
	if err != nil {
		errStr = err.Error()
	}
	return madmin.TraceInfo{
		TraceType: madmin.TraceOS,
		Time:      startTime,
		NodeName:  globalLocalNodeName,
		FuncName:  "os." + s.String(),
		Duration:  duration,
		Path:      path,
		Error:     errStr,
	}
}

func updateOSMetrics(s osMetric, paths ...string) func(err error) {
	if globalTrace.NumSubscribers(madmin.TraceOS) == 0 {
		return func(err error) { globalOSMetrics.time(s) }
	}

	startTime := time.Now()
	return func(err error) {
		duration := time.Since(startTime)
		globalOSMetrics.incTime(s, duration)
		globalTrace.Publish(osTrace(s, startTime, duration, strings.Join(paths, " -> "), err))
	}
}

// RemoveAll captures time taken to call the underlying os.RemoveAll
func RemoveAll(dirPath string) (err error) {
	defer updateOSMetrics(osMetricRemoveAll, dirPath)(err)
	return os.RemoveAll(dirPath)
}

// Mkdir captures time taken to call os.Mkdir
func Mkdir(dirPath string, mode os.FileMode) (err error) {
	defer updateOSMetrics(osMetricMkdir, dirPath)(err)
	return os.Mkdir(dirPath, mode)
}

// MkdirAll captures time taken to call os.MkdirAll
func MkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
	defer updateOSMetrics(osMetricMkdirAll, dirPath)(err)
	return osMkdirAll(dirPath, mode, baseDir)
}

// Rename captures time taken to call os.Rename
func Rename(src, dst string) (err error) {
	defer updateOSMetrics(osMetricRename, src, dst)(err)
	return RenameSys(src, dst)
}

// OpenFile captures time taken to call os.OpenFile
func OpenFile(name string, flag int, perm os.FileMode) (f *os.File, err error) {
	switch flag & writeMode {
	case writeMode:
		defer updateOSMetrics(osMetricOpenFileW, name)(err)
	default:
		defer updateOSMetrics(osMetricOpenFileR, name)(err)
	}
	return os.OpenFile(name, flag, perm)
}

// Access captures time taken to call syscall.Access()
// on windows, plan9 and solaris syscall.Access uses
// os.Lstat()
func Access(name string) (err error) {
	defer updateOSMetrics(osMetricAccess, name)(err)
	return access(name)
}

// Open captures time taken to call os.Open
func Open(name string) (f *os.File, err error) {
	defer updateOSMetrics(osMetricOpen, name)(err)
	return os.Open(name)
}

// OpenFileDirectIO captures time taken to call disk.OpenFileDirectIO
func OpenFileDirectIO(name string, flag int, perm os.FileMode) (f *os.File, err error) {
	defer updateOSMetrics(osMetricOpenFileDirectIO, name)(err)
	return disk.OpenFileDirectIO(name, flag, perm)
}

// Lstat captures time taken to call os.Lstat
func Lstat(name string) (info os.FileInfo, err error) {
	defer updateOSMetrics(osMetricLstat, name)(err)
	return os.Lstat(name)
}

// Remove captures time taken to call os.Remove
func Remove(deletePath string) (err error) {
	defer updateOSMetrics(osMetricRemove, deletePath)(err)
	return os.Remove(deletePath)
}

// Stat captures time taken to call os.Stat
func Stat(name string) (info os.FileInfo, err error) {
	defer updateOSMetrics(osMetricStat, name)(err)
	return os.Stat(name)
}

// Create captures time taken to call os.Create
func Create(name string) (f *os.File, err error) {
	defer updateOSMetrics(osMetricCreate, name)(err)
	return os.Create(name)
}

// Fdatasync captures time taken to call Fdatasync
func Fdatasync(f *os.File) (err error) {
	fn := ""
	if f != nil {
		fn = f.Name()
	}
	defer updateOSMetrics(osMetricFdatasync, fn)(err)
	return disk.Fdatasync(f)
}

// report returns all os metrics.
func (o *osMetrics) report() madmin.OSMetrics {
	var m madmin.OSMetrics
	m.CollectedAt = time.Now()
	m.LifeTimeOps = make(map[string]uint64, osMetricLast)
	for i := osMetric(0); i < osMetricLast; i++ {
		if n := atomic.LoadUint64(&o.operations[i]); n > 0 {
			m.LifeTimeOps[i.String()] = n
		}
	}
	if len(m.LifeTimeOps) == 0 {
		m.LifeTimeOps = nil
	}

	m.LastMinute.Operations = make(map[string]madmin.TimedAction, osMetricLast)
	for i := osMetric(0); i < osMetricLast; i++ {
		lm := o.latency[i].total()
		if lm.N > 0 {
			m.LastMinute.Operations[i.String()] = lm.asTimedAction()
		}
	}
	if len(m.LastMinute.Operations) == 0 {
		m.LastMinute.Operations = nil
	}

	return m
}