From cc8a8cb877756e1f9449f0ef27de256b38a54799 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 10 Jul 2017 18:14:48 -0700 Subject: [PATCH] posix: Check for min disk space and inodes (#4618) This is needed such that we don't start or allow writing to a posix disk which doesn't have minimum total disk space available. One part fix for #4617 --- cmd/fs-v1.go | 16 +++++-- cmd/posix.go | 95 ++++++++++++++++++++++++++++++----------- cmd/posix_test.go | 105 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 28 deletions(-) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 74e6a45ac..960a905ca 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -103,6 +103,16 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { } } + di, err := getDiskInfo(preparePath(fsPath)) + if err != nil { + return nil, err + } + + // Check if disk has minimum required total space. + if err = checkDiskMinTotal(di); err != nil { + return nil, err + } + // Assign a new UUID for FS minio mode. Each server instance // gets its own UUID for temporary file transaction. fsUUID := mustGetUUID() @@ -138,14 +148,12 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { fs.fsFormatRlk = rlk // Initialize and load bucket policies. - err = initBucketPolicies(fs) - if err != nil { + if err = initBucketPolicies(fs); err != nil { return nil, fmt.Errorf("Unable to load all bucket policies. %s", err) } // Initialize a new event notifier. - err = initEventNotifier(fs) - if err != nil { + if err = initEventNotifier(fs); err != nil { return nil, fmt.Errorf("Unable to initialize event notification. %s", err) } diff --git a/cmd/posix.go b/cmd/posix.go index ac0c2c57d..d0d99e08d 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -35,9 +35,11 @@ import ( ) const ( - fsMinFreeSpace = 1 * humanize.GiByte // Min 1GiB free space. - fsMinFreeInodes = 10000 // Min 10000. - maxAllowedIOError = 5 + diskMinFreeSpace = 1 * humanize.GiByte // Min 1GiB free space. + diskMinTotalSpace = diskMinFreeSpace // Min 1GiB total space. + diskMinFreeInodes = 10000 // Min 10000 free inodes. + diskMinTotalInodes = diskMinFreeInodes // Min 10000 total inodes. + maxAllowedIOError = 5 ) // posix - implements StorageAPI interface. @@ -108,7 +110,7 @@ func newPosix(path string) (StorageAPI, error) { if err != nil { return nil, err } - fs := &posix{ + st := &posix{ diskPath: diskPath, // 1MiB buffer pool for posix internal operations. pool: sync.Pool{ @@ -131,7 +133,19 @@ func newPosix(path string) (StorageAPI, error) { return nil, err } } - return fs, nil + + di, err := getDiskInfo(preparePath(diskPath)) + if err != nil { + return nil, err + } + + // Check if disk has minimum required total space. + if err = checkDiskMinTotal(di); err != nil { + return nil, err + } + + // Success. + return st, nil } // getDiskInfo returns given disk information. @@ -155,11 +169,58 @@ var ignoreDiskFreeOS = []string{ globalSolarisOSName, } +// check if disk total has minimum required size. +func checkDiskMinTotal(di disk.Info) (err error) { + // Remove 5% from total space for cumulative disk space + // used for journalling, inodes etc. + totalDiskSpace := float64(di.Total) * 0.95 + if int64(totalDiskSpace) <= diskMinTotalSpace { + return errDiskFull + } + + // Some filesystems do not implement a way to provide total inodes available, instead + // inodes are allocated based on available disk space. For example CephDISK, StoreNext CVDISK, + // AzureFile driver. Allow for the available disk to be separately validated and we will + // validate inodes only if total inodes are provided by the underlying filesystem. + if di.Files != 0 && di.FSType != "NFS" { + totalFiles := int64(di.Files) + if totalFiles <= diskMinTotalInodes { + return errDiskFull + } + } + + return nil +} + +// check if disk free has minimum required size. +func checkDiskMinFree(di disk.Info) error { + // Remove 5% from free space for cumulative disk space used for journalling, inodes etc. + availableDiskSpace := float64(di.Free) * 0.95 + if int64(availableDiskSpace) <= diskMinFreeSpace { + return errDiskFull + } + + // Some filesystems do not implement a way to provide total inodes available, instead inodes + // are allocated based on available disk space. For example CephDISK, StoreNext CVDISK, AzureFile driver. + // Allow for the available disk to be separately validate and we will validate inodes only if + // total inodes are provided by the underlying filesystem. + if di.Files != 0 && di.FSType != "NFS" { + availableFiles := int64(di.Ffree) + if availableFiles <= diskMinFreeInodes { + return errDiskFull + } + } + + // Success. + return nil +} + // checkDiskFree verifies if disk path has sufficient minimum free disk space and files. func checkDiskFree(diskPath string, neededSpace int64) (err error) { // We don't validate disk space or inode utilization on windows. - // Each windows calls to 'GetVolumeInformationW' takes around 3-5seconds. - // And StatFS is not supported by Go for solaris and netbsd. + // Each windows call to 'GetVolumeInformationW' takes around + // 3-5seconds. And StatDISK is not supported by Go for solaris + // and netbsd. if contains(ignoreDiskFreeOS, runtime.GOOS) { return nil } @@ -170,29 +231,15 @@ func checkDiskFree(diskPath string, neededSpace int64) (err error) { return err } - // Remove 5% from free space for cumulative disk space used for journalling, inodes etc. - availableDiskSpace := float64(di.Free) * 0.95 - if int64(availableDiskSpace) <= fsMinFreeSpace { - return errDiskFull - } - - // Some filesystems do not implement a way to provide total inodes available, instead inodes - // are allocated based on available disk space. For example CephFS, StoreNext CVFS, AzureFile driver. - // Allow for the available disk to be separately validate and we will validate inodes only if - // total inodes are provided by the underlying filesystem. - if di.Files != 0 && di.FSType != "NFS" { - availableFiles := int64(di.Ffree) - if availableFiles <= fsMinFreeInodes { - return errDiskFull - } + if err = checkDiskMinFree(di); err != nil { + return err } // Check if we have enough space to store data - if neededSpace > int64(availableDiskSpace) { + if neededSpace > int64(float64(di.Free)*0.95) { return errDiskFull } - // Success. return nil } diff --git a/cmd/posix_test.go b/cmd/posix_test.go index 3ef44dc04..11023be96 100644 --- a/cmd/posix_test.go +++ b/cmd/posix_test.go @@ -29,6 +29,8 @@ import ( "syscall" "testing" + "github.com/minio/minio/pkg/disk" + "golang.org/x/crypto/blake2b" ) @@ -1669,3 +1671,106 @@ func TestPosixStatFile(t *testing.T) { } } } + +// Checks for restrictions for min total disk space and inodes. +func TestCheckDiskTotalMin(t *testing.T) { + testCases := []struct { + diskInfo disk.Info + err error + }{ + // Test 1 - when fstype is nfs. + { + diskInfo: disk.Info{ + Total: diskMinTotalSpace * 3, + FSType: "NFS", + }, + err: nil, + }, + // Test 2 - when fstype is xfs and total inodes are small. + { + diskInfo: disk.Info{ + Total: diskMinTotalSpace * 3, + FSType: "XFS", + Files: 9999, + }, + err: errDiskFull, + }, + // Test 3 - when fstype is btrfs and total inodes is empty. + { + diskInfo: disk.Info{ + Total: diskMinTotalSpace * 3, + FSType: "BTRFS", + Files: 0, + }, + err: nil, + }, + // Test 4 - when fstype is xfs and total disk space is really small. + { + diskInfo: disk.Info{ + Total: diskMinTotalSpace - diskMinTotalSpace/1024, + FSType: "XFS", + Files: 9999, + }, + err: errDiskFull, + }, + } + + // Validate all cases. + for i, test := range testCases { + if err := checkDiskMinTotal(test.diskInfo); test.err != err { + t.Errorf("Test %d: Expected error %s, got %s", i+1, test.err, err) + } + } +} + +// Checks for restrictions for min free disk space and inodes. +func TestCheckDiskFreeMin(t *testing.T) { + testCases := []struct { + diskInfo disk.Info + err error + }{ + // Test 1 - when fstype is nfs. + { + diskInfo: disk.Info{ + Free: diskMinTotalSpace * 3, + FSType: "NFS", + }, + err: nil, + }, + // Test 2 - when fstype is xfs and total inodes are small. + { + diskInfo: disk.Info{ + Free: diskMinTotalSpace * 3, + FSType: "XFS", + Files: 9999, + Ffree: 9999, + }, + err: errDiskFull, + }, + // Test 3 - when fstype is btrfs and total inodes are empty. + { + diskInfo: disk.Info{ + Free: diskMinTotalSpace * 3, + FSType: "BTRFS", + Files: 0, + }, + err: nil, + }, + // Test 4 - when fstype is xfs and total disk space is really small. + { + diskInfo: disk.Info{ + Free: diskMinTotalSpace - diskMinTotalSpace/1024, + FSType: "XFS", + Files: 9999, + }, + err: errDiskFull, + }, + } + + // Validate all cases. + for i, test := range testCases { + if err := checkDiskMinFree(test.diskInfo); test.err != err { + t.Errorf("Test %d: Expected error %s, got %s", i+1, test.err, err) + } + } +}