rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
2021-06-01 14:59:40 -07:00
committed by GitHub
parent bf87c4b1e4
commit 1f262daf6f
540 changed files with 757 additions and 778 deletions

View File

@@ -0,0 +1,42 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
"github.com/ncw/directio"
"golang.org/x/sys/unix"
)
// OpenFileDirectIO - bypass kernel cache.
func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
return directio.OpenFile(filePath, flag, perm)
}
// DisableDirectIO - disables directio mode.
func DisableDirectIO(f *os.File) error {
fd := f.Fd()
_, err := unix.FcntlInt(fd, unix.F_NOCACHE, 0)
return err
}
// AlignedBlock - pass through to directio implementation.
func AlignedBlock(BlockSize int) []byte {
return directio.AlignedBlock(BlockSize)
}

View File

@@ -0,0 +1,50 @@
// +build linux netbsd freebsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
"syscall"
"github.com/ncw/directio"
"golang.org/x/sys/unix"
)
// OpenFileDirectIO - bypass kernel cache.
func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
return directio.OpenFile(filePath, flag, perm)
}
// DisableDirectIO - disables directio mode.
func DisableDirectIO(f *os.File) error {
fd := f.Fd()
flag, err := unix.FcntlInt(fd, unix.F_GETFL, 0)
if err != nil {
return err
}
flag = flag & ^(syscall.O_DIRECT)
_, err = unix.FcntlInt(fd, unix.F_SETFL, flag)
return err
}
// AlignedBlock - pass through to directio implementation.
func AlignedBlock(BlockSize int) []byte {
return directio.AlignedBlock(BlockSize)
}

View File

@@ -0,0 +1,65 @@
// +build !linux,!netbsd,!freebsd,!darwin
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
)
// OpenBSD, Windows, and illumos do not support O_DIRECT.
// On Windows there is no documentation on disabling O_DIRECT.
// For these systems we do not attempt to build the 'directio' dependency since
// the O_DIRECT symbol may not be exposed resulting in a failed build.
//
//
// On illumos an explicit O_DIRECT flag is not necessary for two primary
// reasons. Note that ZFS is effectively the default filesystem on illumos
// systems.
//
// One benefit of using DirectIO on Linux is that the page cache will not be
// polluted with single-access data. The ZFS read cache (ARC) is scan-resistant
// so there is no risk of polluting the entire cache with data accessed once.
// Another goal of DirectIO is to minimize the mutation of data by the kernel
// before issuing IO to underlying devices. ZFS users often enable features like
// compression and checksumming which currently necessitates mutating data in
// the kernel.
//
// DirectIO semantics for a filesystem like ZFS would be quite different than
// the semantics on filesystems like XFS, and these semantics are not
// implemented at this time.
// For more information on why typical DirectIO semantics do not apply to ZFS
// see this ZFS-on-Linux commit message:
// https://github.com/openzfs/zfs/commit/a584ef26053065f486d46a7335bea222cb03eeea
// OpenFileDirectIO wrapper around os.OpenFile nothing special
func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(filePath, flag, perm)
}
// DisableDirectIO is a no-op
func DisableDirectIO(f *os.File) error {
return nil
}
// AlignedBlock simply returns an unaligned buffer
// for systems that do not support DirectIO.
func AlignedBlock(BlockSize int) []byte {
return make([]byte, BlockSize)
}

33
internal/disk/disk.go Normal file
View File

@@ -0,0 +1,33 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
// Info stat fs struct is container which holds following values
// Total - total size of the volume / disk
// Free - free size of the volume / disk
// Files - total inodes available
// Ffree - free inodes available
// FSType - file system type
type Info struct {
Total uint64
Free uint64
Used uint64
Files uint64
Ffree uint64
FSType string
}

View File

@@ -0,0 +1,45 @@
// +build !netbsd,!solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk_test
import (
"io/ioutil"
"os"
"testing"
"github.com/minio/minio/internal/disk"
)
func TestFree(t *testing.T) {
path, err := ioutil.TempDir(os.TempDir(), "minio-")
defer os.RemoveAll(path)
if err != nil {
t.Fatal(err)
}
di, err := disk.GetInfo(path)
if err != nil {
t.Fatal(err)
}
if di.FSType == "UNKNOWN" {
t.Error("Unexpected FSType", di.FSType)
}
}

View File

@@ -0,0 +1,40 @@
// +build !windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"syscall"
)
// SameDisk reports whether di1 and di2 describe the same disk.
func SameDisk(disk1, disk2 string) (bool, error) {
st1 := syscall.Stat_t{}
st2 := syscall.Stat_t{}
if err := syscall.Stat(disk1, &st1); err != nil {
return false, err
}
if err := syscall.Stat(disk2, &st2); err != nil {
return false, err
}
return st1.Dev == st2.Dev, nil
}

View File

@@ -0,0 +1,25 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
// SameDisk reports whether di1 and di2 describe the same disk.
func SameDisk(disk1, disk2 string) (bool, error) {
return false, nil
}

View File

@@ -0,0 +1,39 @@
// +build linux
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
"syscall"
)
// Fdatasync - fdatasync() is similar to fsync(), but does not flush modified metadata
// unless that metadata is needed in order to allow a subsequent data retrieval
// to be correctly handled. For example, changes to st_atime or st_mtime
// (respectively, time of last access and time of last modification; see inode(7))
// do not require flushing because they are not necessary for a subsequent data
// read to be handled correctly. On the other hand, a change to the file size
// (st_size, as made by say ftruncate(2)), would require a metadata flush.
//
// The aim of fdatasync() is to reduce disk activity for applications that
// do not require all metadata to be synchronized with the disk.
func Fdatasync(f *os.File) error {
return syscall.Fdatasync(int(f.Fd()))
}

View File

@@ -0,0 +1,30 @@
// +build freebsd netbsd openbsd darwin
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
"syscall"
)
// Fdatasync is fsync on freebsd/darwin
func Fdatasync(f *os.File) error {
return syscall.Fsync(int(f.Fd()))
}

View File

@@ -0,0 +1,29 @@
// +build !linux,!netbsd,!freebsd,!darwin,!openbsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"os"
)
// Fdatasync is a no-op
func Fdatasync(f *os.File) error {
return nil
}

146
internal/disk/health.go Normal file
View File

@@ -0,0 +1,146 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"context"
"fmt"
"os"
"time"
"github.com/dustin/go-humanize"
"github.com/minio/madmin-go"
"github.com/montanaflynn/stats"
)
// GetHealthInfo about the drive
func GetHealthInfo(ctx context.Context, drive, fsPath string) (madmin.DiskLatency, madmin.DiskThroughput, error) {
// Create a file with O_DIRECT flag, choose default umask and also make sure
// we are exclusively writing to a new file using O_EXCL.
w, err := OpenFileDirectIO(fsPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666)
if err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
defer func() {
w.Close()
os.Remove(fsPath)
}()
blockSize := 4 * humanize.MiByte
fileSize := 256 * humanize.MiByte
latencies := make([]float64, fileSize/blockSize)
throughputs := make([]float64, fileSize/blockSize)
data := AlignedBlock(blockSize)
for i := 0; i < (fileSize / blockSize); i++ {
if ctx.Err() != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, ctx.Err()
}
startTime := time.Now()
if n, err := w.Write(data); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
} else if n != blockSize {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, fmt.Errorf("Expected to write %d, but only wrote %d", blockSize, n)
}
latencyInSecs := time.Since(startTime).Seconds()
latencies[i] = latencyInSecs
}
// Sync every full writes fdatasync
Fdatasync(w)
for i := range latencies {
throughput := float64(blockSize) / latencies[i]
throughputs[i] = throughput
}
var avgLatency float64
var percentile50Latency float64
var percentile90Latency float64
var percentile99Latency float64
var minLatency float64
var maxLatency float64
var avgThroughput float64
var percentile50Throughput float64
var percentile90Throughput float64
var percentile99Throughput float64
var minThroughput float64
var maxThroughput float64
if avgLatency, err = stats.Mean(latencies); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile50Latency, err = stats.Percentile(latencies, 50); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile90Latency, err = stats.Percentile(latencies, 90); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile99Latency, err = stats.Percentile(latencies, 99); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if maxLatency, err = stats.Max(latencies); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if minLatency, err = stats.Min(latencies); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
l := madmin.DiskLatency{
Avg: avgLatency,
Percentile50: percentile50Latency,
Percentile90: percentile90Latency,
Percentile99: percentile99Latency,
Min: minLatency,
Max: maxLatency,
}
if avgThroughput, err = stats.Mean(throughputs); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile50Throughput, err = stats.Percentile(throughputs, 50); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile90Throughput, err = stats.Percentile(throughputs, 90); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if percentile99Throughput, err = stats.Percentile(throughputs, 99); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if maxThroughput, err = stats.Max(throughputs); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
if minThroughput, err = stats.Min(throughputs); err != nil {
return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
}
t := madmin.DiskThroughput{
Avg: avgThroughput,
Percentile50: percentile50Throughput,
Percentile90: percentile90Throughput,
Percentile99: percentile99Throughput,
Min: minThroughput,
Max: maxThroughput,
}
return l, t, nil
}

View File

@@ -0,0 +1,29 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import "runtime"
// IsRootDisk returns if diskPath belongs to root-disk, i.e the disk mounted at "/"
func IsRootDisk(diskPath string, rootDisk string) (bool, error) {
if runtime.GOOS == "windows" {
// On windows this function is not implemented.
return false, nil
}
return SameDisk(diskPath, rootDisk)
}

47
internal/disk/stat_bsd.go Normal file
View File

@@ -0,0 +1,47 @@
// +build darwin dragonfly
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"syscall"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Bsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Bsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Fstypename[:]),
}
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,47 @@
// +build freebsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"syscall"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - uint64(s.Bavail)
info = Info{
Total: uint64(s.Bsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Bsize) * uint64(s.Bavail),
Files: s.Files,
Ffree: uint64(s.Ffree),
FSType: getFSType(s.Fstypename[:]),
}
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,52 @@
// +build linux,!s390x,!arm,!386
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"syscall"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
//nolint:unconvert
FSType: getFSType(int64(s.Type)),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035
// XFS can show wrong values at times error out
// in such scenarios.
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,82 @@
// +build linux,arm linux,386
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"strconv"
"syscall"
)
// fsType2StringMap - list of filesystems supported on linux
var fsType2StringMap = map[string]string{
"1021994": "TMPFS",
"137d": "EXT",
"4244": "HFS",
"4d44": "MSDOS",
"52654973": "REISERFS",
"5346544e": "NTFS",
"58465342": "XFS",
"61756673": "AUFS",
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
"794c7630": "overlayfs",
"2fc12fc1": "zfs",
"ff534d42": "cifs",
"53464846": "wslfs",
}
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(ftype int32) string {
fsTypeHex := strconv.FormatInt(int64(ftype), 16)
fsTypeString, ok := fsType2StringMap[fsTypeHex]
if !ok {
return "UNKNOWN"
}
return fsTypeString
}
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Type),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035
// XFS can show wrong values at times error out
// in such scenarios.
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,82 @@
// +build linux,s390x
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"strconv"
"syscall"
)
// fsType2StringMap - list of filesystems supported on linux
var fsType2StringMap = map[string]string{
"1021994": "TMPFS",
"137d": "EXT",
"4244": "HFS",
"4d44": "MSDOS",
"52654973": "REISERFS",
"5346544e": "NTFS",
"58465342": "XFS",
"61756673": "AUFS",
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
"794c7630": "overlayfs",
"2fc12fc1": "zfs",
"ff534d42": "cifs",
"53464846": "wslfs",
}
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(ftype uint32) string {
fsTypeHex := strconv.FormatUint(uint64(ftype), 16)
fsTypeString, ok := fsType2StringMap[fsTypeHex]
if !ok {
return "UNKNOWN"
}
return fsTypeString
}
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := s.Bfree - s.Bavail
info = Info{
Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks),
Free: uint64(s.Frsize) * s.Bavail,
Files: s.Files,
Ffree: s.Ffree,
FSType: getFSType(s.Type),
}
// Check for overflows.
// https://github.com/minio/minio/issues/8035
// XFS can show wrong values at times error out
// in such scenarios.
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,47 @@
// +build netbsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"golang.org/x/sys/unix"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := unix.Statvfs_t{}
if err = unix.Statvfs(path, &s); err != nil {
return Info{}, err
}
reservedBlocks := uint64(s.Bfree) - uint64(s.Bavail)
info = Info{
Total: uint64(s.Frsize) * (uint64(s.Blocks) - reservedBlocks),
Free: uint64(s.Frsize) * uint64(s.Bavail),
Files: uint64(s.Files),
Ffree: uint64(s.Ffree),
FSType: string(s.Fstypename[:]),
}
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,47 @@
// +build openbsd
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"syscall"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := syscall.Statfs_t{}
err = syscall.Statfs(path, &s)
if err != nil {
return Info{}, err
}
reservedBlocks := uint64(s.F_bfree) - uint64(s.F_bavail)
info = Info{
Total: uint64(s.F_bsize) * (uint64(s.F_blocks) - reservedBlocks),
Free: uint64(s.F_bsize) * uint64(s.F_bavail),
Files: uint64(s.F_files),
Ffree: uint64(s.F_ffree),
FSType: getFSType(s.F_fstypename[:]),
}
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,47 @@
// +build solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"golang.org/x/sys/unix"
)
// GetInfo returns total and free bytes available in a directory, e.g. `/`.
func GetInfo(path string) (info Info, err error) {
s := unix.Statvfs_t{}
if err = unix.Statvfs(path, &s); err != nil {
return Info{}, err
}
reservedBlocks := uint64(s.Bfree) - uint64(s.Bavail)
info = Info{
Total: uint64(s.Frsize) * (uint64(s.Blocks) - reservedBlocks),
Free: uint64(s.Frsize) * uint64(s.Bavail),
Files: uint64(s.Files),
Ffree: uint64(s.Ffree),
FSType: getFSType(s.Fstr[:]),
}
if info.Free > info.Total {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
}
info.Used = info.Total - info.Free
return info, nil
}

View File

@@ -0,0 +1,107 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"fmt"
"os"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var (
kernel32 = windows.NewLazySystemDLL("kernel32.dll")
// GetDiskFreeSpaceEx - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
// Retrieves information about the amount of space that is available on a disk volume,
// which is the total amount of space, the total amount of free space, and the total
// amount of free space available to the user that is associated with the calling thread.
GetDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW")
// GetDiskFreeSpace - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364935(v=vs.85).aspx
// Retrieves information about the specified disk, including the amount of free space on the disk.
GetDiskFreeSpace = kernel32.NewProc("GetDiskFreeSpaceW")
)
// GetInfo returns total and free bytes available in a directory, e.g. `C:\`.
// It returns free space available to the user (including quota limitations)
//
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
func GetInfo(path string) (info Info, err error) {
// Stat to know if the path exists.
if _, err = os.Stat(path); err != nil {
return Info{}, err
}
lpFreeBytesAvailable := int64(0)
lpTotalNumberOfBytes := int64(0)
lpTotalNumberOfFreeBytes := int64(0)
// Extract values safely
// BOOL WINAPI GetDiskFreeSpaceEx(
// _In_opt_ LPCTSTR lpDirectoryName,
// _Out_opt_ PULARGE_INTEGER lpFreeBytesAvailable,
// _Out_opt_ PULARGE_INTEGER lpTotalNumberOfBytes,
// _Out_opt_ PULARGE_INTEGER lpTotalNumberOfFreeBytes
// );
_, _, _ = GetDiskFreeSpaceEx.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
if uint64(lpTotalNumberOfFreeBytes) > uint64(lpTotalNumberOfBytes) {
return info, fmt.Errorf("detected free space (%d) > total disk space (%d), fs corruption at (%s). please run 'fsck'",
uint64(lpTotalNumberOfFreeBytes), uint64(lpTotalNumberOfBytes), path)
}
info = Info{
Total: uint64(lpTotalNumberOfBytes),
Free: uint64(lpTotalNumberOfFreeBytes),
Used: uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes),
FSType: getFSType(path),
}
// Return values of GetDiskFreeSpace()
lpSectorsPerCluster := uint32(0)
lpBytesPerSector := uint32(0)
lpNumberOfFreeClusters := uint32(0)
lpTotalNumberOfClusters := uint32(0)
// Extract values safely
// BOOL WINAPI GetDiskFreeSpace(
// _In_ LPCTSTR lpRootPathName,
// _Out_ LPDWORD lpSectorsPerCluster,
// _Out_ LPDWORD lpBytesPerSector,
// _Out_ LPDWORD lpNumberOfFreeClusters,
// _Out_ LPDWORD lpTotalNumberOfClusters
// );
_, _, _ = GetDiskFreeSpace.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
uintptr(unsafe.Pointer(&lpSectorsPerCluster)),
uintptr(unsafe.Pointer(&lpBytesPerSector)),
uintptr(unsafe.Pointer(&lpNumberOfFreeClusters)),
uintptr(unsafe.Pointer(&lpTotalNumberOfClusters)))
info.Files = uint64(lpTotalNumberOfClusters)
info.Ffree = uint64(lpNumberOfFreeClusters)
return info, nil
}

29
internal/disk/type_bsd.go Normal file
View File

@@ -0,0 +1,29 @@
// +build darwin freebsd dragonfly openbsd solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(fstype []int8) string {
b := make([]byte, len(fstype))
for i, v := range fstype {
b[i] = byte(v)
}
return string(b)
}

View File

@@ -0,0 +1,52 @@
// +build linux,!s390x,!arm,!386
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import "strconv"
// fsType2StringMap - list of filesystems supported on linux
var fsType2StringMap = map[string]string{
"1021994": "TMPFS",
"137d": "EXT",
"4244": "HFS",
"4d44": "MSDOS",
"52654973": "REISERFS",
"5346544e": "NTFS",
"58465342": "XFS",
"61756673": "AUFS",
"6969": "NFS",
"ef51": "EXT2OLD",
"ef53": "EXT4",
"f15f": "ecryptfs",
"794c7630": "overlayfs",
"2fc12fc1": "zfs",
"ff534d42": "cifs",
"53464846": "wslfs",
}
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(ftype int64) string {
fsTypeHex := strconv.FormatInt(ftype, 16)
fsTypeString, ok := fsType2StringMap[fsTypeHex]
if !ok {
return "UNKNOWN"
}
return fsTypeString
}

View File

@@ -0,0 +1,63 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package disk
import (
"path/filepath"
"syscall"
"unsafe"
)
var (
// GetVolumeInformation provides windows drive volume information.
GetVolumeInformation = kernel32.NewProc("GetVolumeInformationW")
)
// getFSType returns the filesystem type of the underlying mounted filesystem
func getFSType(path string) string {
volumeNameSize, nFileSystemNameSize := uint32(260), uint32(260)
var lpVolumeSerialNumber uint32
var lpFileSystemFlags, lpMaximumComponentLength uint32
var lpFileSystemNameBuffer, volumeName [260]uint16
var ps = syscall.StringToUTF16Ptr(filepath.VolumeName(path))
// Extract values safely
// BOOL WINAPI GetVolumeInformation(
// _In_opt_ LPCTSTR lpRootPathName,
// _Out_opt_ LPTSTR lpVolumeNameBuffer,
// _In_ DWORD nVolumeNameSize,
// _Out_opt_ LPDWORD lpVolumeSerialNumber,
// _Out_opt_ LPDWORD lpMaximumComponentLength,
// _Out_opt_ LPDWORD lpFileSystemFlags,
// _Out_opt_ LPTSTR lpFileSystemNameBuffer,
// _In_ DWORD nFileSystemNameSize
// );
_, _, _ = GetVolumeInformation.Call(uintptr(unsafe.Pointer(ps)),
uintptr(unsafe.Pointer(&volumeName)),
uintptr(volumeNameSize),
uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
uintptr(unsafe.Pointer(&lpFileSystemFlags)),
uintptr(unsafe.Pointer(&lpFileSystemNameBuffer)),
uintptr(nFileSystemNameSize))
return syscall.UTF16ToString(lpFileSystemNameBuffer[:])
}