MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2024-12-24 22:25:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: optionally enable tracing posix calls

Browse Source
This commit is contained in:
Ritesh H Shukla 2021-03-23 14:51:27 -07:00 committed by Harshavardhana
parent 8adfeb0d84
commit 6a2ed44095
5 changed files with 230 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cmd/config/constants.go
View File

@ -23,21 +23,23 @@ const (
// Top level common ENVs
const (
EnvAccessKey = "MINIO_ACCESS_KEY"
EnvSecretKey = "MINIO_SECRET_KEY"
EnvRootUser = "MINIO_ROOT_USER"
EnvRootPassword = "MINIO_ROOT_PASSWORD"
EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
EnvRootUserOld = "MINIO_ROOT_USER_OLD"
EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
EnvBrowser = "MINIO_BROWSER"
EnvDomain = "MINIO_DOMAIN"
EnvRegionName = "MINIO_REGION_NAME"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvFSOSync = "MINIO_FS_OSYNC"
EnvArgs = "MINIO_ARGS"
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
EnvAccessKey = "MINIO_ACCESS_KEY"
EnvSecretKey = "MINIO_SECRET_KEY"
EnvRootUser = "MINIO_ROOT_USER"
EnvRootPassword = "MINIO_ROOT_PASSWORD"
EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
EnvRootUserOld = "MINIO_ROOT_USER_OLD"
EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
EnvBrowser = "MINIO_BROWSER"
EnvDomain = "MINIO_DOMAIN"
EnvRegionName = "MINIO_REGION_NAME"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvFSOSync = "MINIO_FS_OSYNC"
EnvArgs = "MINIO_ARGS"
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
EnvLogPosixTimes = "MINIO_LOG_POSIX_TIMES"
EnvLogPosixThresholdInMS = "MINIO_LOG_POSIX_THRESHOLD_MS"
EnvUpdate = "MINIO_UPDATE"

169
cmd/os-instrumented.go Normal file
View File

@ -0,0 +1,169 @@
/*
* MinIO Cloud Storage, (C) 2021 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"fmt"
"os"
"strings"
"time"
"github.com/minio/minio/cmd/config"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/env"
)
var (
logTime bool = false
threshold time.Duration
)
func init() {
logTime = env.IsSet(config.EnvLogPosixTimes)
t, _ := env.GetInt(
config.EnvLogPosixThresholdInMS,
100,
)
threshold = time.Duration(t) * time.Millisecond
}
func getThreshold() time.Duration {
return threshold
}
func collectLogTime() bool {
return logTime
}
func reportTime(name *strings.Builder, startTime time.Time) {
delta := time.Since(startTime)
if delta > getThreshold() {
name.WriteString(" ")
name.WriteString(delta.String())
fmt.Println(name.String())
}
}
// RemoveAll captures time taken to call the underlying os.RemoveAll
func RemoveAll(dirPath string) error {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.RemoveAll: ")
s.WriteString(dirPath)
defer reportTime(&s, startTime)
}
return os.RemoveAll(dirPath)
}
// MkdirAll captures time taken to call os.MkdirAll
func MkdirAll(dirPath string, mode os.FileMode) error {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.MkdirAll: ")
s.WriteString(dirPath)
defer reportTime(&s, startTime)
}
return os.MkdirAll(dirPath, mode)
}
// Rename captures time taken to call os.Rename
func Rename(src, dst string) error {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.Rename: ")
s.WriteString(src)
s.WriteString(" to ")
s.WriteString(dst)
defer reportTime(&s, startTime)
}
return os.Rename(src, dst)
}
// OpenFile captures time taken to call os.OpenFile
func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.OpenFile: ")
s.WriteString(name)
defer reportTime(&s, startTime)
}
return os.OpenFile(name, flag, perm)
}
// Open captures time taken to call os.Open
func Open(name string) (*os.File, error) {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.Open: ")
s.WriteString(name)
defer reportTime(&s, startTime)
}
return os.Open(name)
}
// OpenFileDirectIO captures time taken to call disk.OpenFileDirectIO
func OpenFileDirectIO(name string, flag int, perm os.FileMode) (*os.File, error) {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("disk.OpenFileDirectIO: ")
s.WriteString(name)
defer reportTime(&s, startTime)
}
return disk.OpenFileDirectIO(name, flag, perm)
}
// Lstat captures time taken to call os.Lstat
func Lstat(name string) (os.FileInfo, error) {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.Lstat: ")
s.WriteString(name)
defer reportTime(&s, startTime)
}
return os.Lstat(name)
}
// Remove captures time taken to call os.Remove
func Remove(deletePath string) error {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.Remove: ")
s.WriteString(deletePath)
defer reportTime(&s, startTime)
}
return os.Remove(deletePath)
}
// Stat captures time taken to call os.Stat
func Stat(name string) (os.FileInfo, error) {
if collectLogTime() {
startTime := time.Now()
var s strings.Builder
s.WriteString("os.Stat: ")
s.WriteString(name)
defer reportTime(&s, startTime)
}
return os.Stat(name)
}

8
cmd/os-readdir_other.go
View File

@ -33,7 +33,7 @@ func readDir(dirPath string) (entries []string, err error) {
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
d, err := os.Open(dirPath)
d, err := Open(dirPath)
if err != nil {
if osErrToFileErr(err) == errFileNotFound {
return nil
@ -58,7 +58,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
}
for _, fi := range fis {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
fi, err = os.Stat(pathJoin(dirPath, fi.Name()))
fi, err = Stat(pathJoin(dirPath, fi.Name()))
if err != nil {
// It got deleted in the meantime, not found
// or returns too many symlinks ignore this
@ -86,7 +86,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
// Return N entries at the directory dirPath. If count is -1, return all entries
func readDirN(dirPath string, count int) (entries []string, err error) {
d, err := os.Open(dirPath)
d, err := Open(dirPath)
if err != nil {
return nil, osErrToFileErr(err)
}
@ -117,7 +117,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
}
for _, fi := range fis {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
fi, err = os.Stat(pathJoin(dirPath, fi.Name()))
fi, err = Stat(pathJoin(dirPath, fi.Name()))
if err != nil {
// It got deleted in the meantime, not found
// or returns too many symlinks ignore this

6
cmd/os-reliable.go
View File

@ -57,7 +57,7 @@ func reliableRemoveAll(dirPath string) (err error) {
i := 0
for {
// Removes all the directories and files.
if err = os.RemoveAll(dirPath); err != nil {
if err = RemoveAll(dirPath); err != nil {
// Retry only for the first retryable error.
if isSysErrNotEmpty(err) && i == 0 {
i++
@ -101,7 +101,7 @@ func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
i := 0
for {
// Creates all the parent directories, with mode 0777 mkdir honors system umask.
if err = os.MkdirAll(dirPath, mode); err != nil {
if err = MkdirAll(dirPath, mode); err != nil {
// Retry only for the first retryable error.
if osIsNotExist(err) && i == 0 {
i++
@ -166,7 +166,7 @@ func reliableRename(srcFilePath, dstFilePath string) (err error) {
i := 0
for {
// After a successful parent directory create attempt a renameAll.
if err = os.Rename(srcFilePath, dstFilePath); err != nil {
if err = Rename(srcFilePath, dstFilePath); err != nil {
// Retry only for the first retryable error.
if osIsNotExist(err) && i == 0 {
i++

74
cmd/xl-storage.go
View File

@ -172,7 +172,7 @@ func getValidPath(path string) (string, error) {
return path, err
}
fi, err := os.Lstat(path)
fi, err := Lstat(path)
if err != nil && !osIsNotExist(err) {
return path, err
}
@ -296,7 +296,7 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
return p, err
}
w.Close()
defer os.Remove(filePath)
defer Remove(filePath)
// Success.
return p, nil
@ -525,11 +525,11 @@ func (s *xlStorage) GetDiskID() (string, error) {
s.Unlock()
formatFile := pathJoin(s.diskPath, minioMetaBucket, formatConfigFile)
fi, err := os.Lstat(formatFile)
fi, err := Lstat(formatFile)
if err != nil {
// If the disk is still not initialized.
if osIsNotExist(err) {
_, err = os.Lstat(s.diskPath)
_, err = Lstat(s.diskPath)
if err == nil {
// Disk is present but missing `format.json`
return "", errUnformattedDisk
@ -560,7 +560,7 @@ func (s *xlStorage) GetDiskID() (string, error) {
if err != nil {
// If the disk is still not initialized.
if osIsNotExist(err) {
_, err = os.Lstat(s.diskPath)
_, err = Lstat(s.diskPath)
if err == nil {
// Disk is present but missing `format.json`
return "", errUnformattedDisk
@ -623,7 +623,7 @@ func (s *xlStorage) MakeVol(ctx context.Context, volume string) error {
return err
}
if _, err := os.Lstat(volumeDir); err != nil {
if _, err := Lstat(volumeDir); err != nil {
// Volume does not exist we proceed to create.
if osIsNotExist(err) {
// Make a volume entry, with mode 0777 mkdir honors system umask.
@ -677,7 +677,7 @@ func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, er
}
// Stat a volume entry.
var st os.FileInfo
st, err = os.Lstat(volumeDir)
st, err = Lstat(volumeDir)
if err != nil {
switch {
case osIsNotExist(err):
@ -708,9 +708,9 @@ func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bo
}
if forceDelete {
err = os.RemoveAll(volumeDir)
err = RemoveAll(volumeDir)
} else {
err = os.Remove(volumeDir)
err = Remove(volumeDir)
}
if err != nil {
@ -736,7 +736,7 @@ func (s *xlStorage) isLeaf(volume string, leafPath string) bool {
return false
}
_, err = os.Lstat(pathJoin(volumeDir, leafPath, xlStorageFormatFile))
_, err = Lstat(pathJoin(volumeDir, leafPath, xlStorageFormatFile))
if err == nil {
return true
}
@ -767,7 +767,7 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i
}
if err != nil {
if err == errFileNotFound {
if _, verr := os.Lstat(volumeDir); verr != nil {
if _, verr := Lstat(volumeDir); verr != nil {
if osIsNotExist(verr) {
return nil, errVolumeNotFound
} else if isSysErrIO(verr) {
@ -942,7 +942,7 @@ func (s *xlStorage) renameLegacyMetadata(volumeDir, path string) (err error) {
}
}()
if err = os.Rename(srcFilePath, dstFilePath); err != nil {
if err = Rename(srcFilePath, dstFilePath); err != nil {
switch {
case isSysErrNotDir(err):
return errFileNotFound
@ -1035,13 +1035,13 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect
if requireDirectIO {
f, err = disk.OpenFileDirectIO(filePath, readMode, 0666)
} else {
f, err = os.OpenFile(filePath, readMode, 0)
f, err = OpenFile(filePath, readMode, 0)
}
if err != nil {
if osIsNotExist(err) {
// Check if the object doesn't exist because its bucket
// is missing in order to return the correct error.
_, err = os.Lstat(volumeDir)
_, err = Lstat(volumeDir)
if err != nil && osIsNotExist(err) {
return nil, errVolumeNotFound
}
@ -1058,7 +1058,7 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect
} else if isSysErrTooManyFiles(err) {
return nil, errTooManyOpenFiles
} else if isSysErrInvalidArg(err) {
st, _ := os.Lstat(filePath)
st, _ := Lstat(filePath)
if st != nil && st.IsDir() {
// Linux returns InvalidArg for directory O_DIRECT
// we need to keep this fallback code to return correct
@ -1128,7 +1128,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of
var n int
// Stat a volume entry.
_, err = os.Lstat(volumeDir)
_, err = Lstat(volumeDir)
if err != nil {
if osIsNotExist(err) {
return 0, errVolumeNotFound
@ -1145,7 +1145,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of
}
// Open the file for reading.
file, err := os.Open(filePath)
file, err := Open(filePath)
if err != nil {
switch {
case osIsNotExist(err):
@ -1223,7 +1223,7 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
return nil, osErrToFileErr(err)
}
w, err := os.OpenFile(filePath, mode|writeMode, 0666)
w, err := OpenFile(filePath, mode|writeMode, 0666)
if err != nil {
// File path cannot be verified since one of the parents is a file.
switch {
@ -1333,12 +1333,12 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
file, err = disk.OpenFileDirectIO(filePath, readMode, 0666)
} else {
// Open the file for reading.
file, err = os.OpenFile(filePath, readMode, 0666)
file, err = OpenFile(filePath, readMode, 0666)
}
if err != nil {
switch {
case osIsNotExist(err):
_, err = os.Lstat(volumeDir)
_, err = Lstat(volumeDir)
if err != nil && osIsNotExist(err) {
return nil, errVolumeNotFound
}
@ -1461,7 +1461,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
return osErrToFileErr(err)
}
w, err := disk.OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666)
w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666)
if err != nil {
return osErrToFileErr(err)
}
@ -1516,7 +1516,7 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string,
}
// Stat a volume entry.
if _, err = os.Lstat(volumeDir); err != nil {
if _, err = Lstat(volumeDir); err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
}
@ -1552,7 +1552,7 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
}
// Stat a volume entry.
if _, err = os.Lstat(volumeDir); err != nil {
if _, err = Lstat(volumeDir); err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
}
@ -1568,7 +1568,7 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
if err = checkPathLength(filePath); err != nil {
return err
}
st, err := os.Lstat(filePath)
st, err := Lstat(filePath)
if err != nil {
return osErrToFileErr(err)
}
@ -1608,7 +1608,7 @@ func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) e
return err
}
st, _ := os.Lstat(filePath)
st, _ := Lstat(filePath)
if st == nil {
if !s.formatLegacy {
return errPathNotFound
@ -1619,7 +1619,7 @@ func (s *xlStorage) CheckFile(ctx context.Context, volume string, path string) e
return err
}
st, _ = os.Lstat(filePathOld)
st, _ = Lstat(filePathOld)
if st == nil {
return errPathNotFound
}
@ -1661,7 +1661,7 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive bool) erro
tmpuuid := mustGetUUID()
err = renameAll(deletePath, pathutil.Join(s.diskPath, minioMetaTmpDeletedBucket, tmpuuid))
} else {
err = os.Remove(deletePath)
err = Remove(deletePath)
}
if err != nil {
switch {
@ -1703,7 +1703,7 @@ func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recu
}
// Stat a volume entry.
_, err = os.Lstat(volumeDir)
_, err = Lstat(volumeDir)
if err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
@ -1739,7 +1739,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
}
// Stat a volume entry.
_, err = os.Lstat(srcVolumeDir)
_, err = Lstat(srcVolumeDir)
if err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
@ -1749,7 +1749,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
return err
}
if _, err = os.Lstat(dstVolumeDir); err != nil {
if _, err = Lstat(dstVolumeDir); err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
} else if isSysErrIO(err) {
@ -1912,7 +1912,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
continue
}
if err = os.Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil {
if err = Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil {
return osErrToFileErr(err)
}
}
@ -1980,7 +1980,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
return err
}
// Stat a volume entry.
_, err = os.Lstat(srcVolumeDir)
_, err = Lstat(srcVolumeDir)
if err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
@ -1989,7 +1989,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
}
return err
}
_, err = os.Lstat(dstVolumeDir)
_, err = Lstat(dstVolumeDir)
if err != nil {
if osIsNotExist(err) {
return errVolumeNotFound
@ -2017,7 +2017,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
// If source is a directory, we expect the destination to be non-existent but we
// we still need to allow overwriting an empty directory since it represents
// an object empty directory.
dirInfo, err := os.Lstat(dstFilePath)
dirInfo, err := Lstat(dstFilePath)
if isSysErrIO(err) {
return errFaultyDisk
}
@ -2029,7 +2029,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
if !dirInfo.IsDir() {
return errFileAccessDenied
}
if err = os.Remove(dstFilePath); err != nil {
if err = Remove(dstFilePath); err != nil {
if isSysErrNotEmpty(err) {
return errFileAccessDenied
}
@ -2051,7 +2051,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
// Open the file for reading.
file, err := os.Open(partPath)
file, err := Open(partPath)
if err != nil {
return osErrToFileErr(err)
}
@ -2124,7 +2124,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
}
// Stat a volume entry.
_, err = os.Lstat(volumeDir)
_, err = Lstat(volumeDir)
if err != nil {
if osIsNotExist(err) {
return errVolumeNotFound