fs: Add safe locking semantics for format.json (#4523)

This patch also reverts previous changes which were
merged for migration to the newer disk format. We will
be bringing these changes in subsequent releases. But
we wish to add protection in this release such that
future release migrations are protected.

Revert "fs: Migration should handle bucketConfigs as regular objects. (#4482)"
This reverts commit 976870a391.

Revert "fs: Migrate object metadata to objects directory. (#4195)"
This reverts commit 76f4f20609.

pkg/lock/lock.go

@@ -19,10 +19,16 @@
 package lock
 
 import (
+	"errors"
 	"os"
 	"sync"
 )
 
+var (
+	// ErrAlreadyLocked is returned if the underlying fd is already locked.
+	ErrAlreadyLocked = errors.New("file already locked")
+)
+
 // RLockedFile represents a read locked file, implements a special
 // closer which only closes the associated *os.File when the ref count.
 // has reached zero, i.e when all the readers have given up their locks.

pkg/lock/lock_nix.go

@@ -24,16 +24,12 @@ import (
 	"syscall"
 )
 
-// LockedOpenFile - initializes a new lock and protects
-// the file from concurrent access across mount points.
-// This implementation doesn't support all the open
-// flags and shouldn't be considered as replacement
-// for os.OpenFile().
-func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
-	var lockType int
+// Internal function implements support for both
+// blocking and non blocking lock type.
+func lockedOpenFile(path string, flag int, perm os.FileMode, lockType int) (*LockedFile, error) {
 	switch flag {
 	case syscall.O_RDONLY:
-		lockType = syscall.LOCK_SH
+		lockType |= syscall.LOCK_SH
 	case syscall.O_WRONLY:
 		fallthrough
 	case syscall.O_RDWR:
@@ -41,7 +37,7 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 	case syscall.O_WRONLY | syscall.O_CREAT:
 		fallthrough
 	case syscall.O_RDWR | syscall.O_CREAT:
-		lockType = syscall.LOCK_EX
+		lockType |= syscall.LOCK_EX
 	default:
 		return nil, fmt.Errorf("Unsupported flag (%d)", flag)
 	}
@@ -53,6 +49,9 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 
 	if err = syscall.Flock(int(f.Fd()), lockType); err != nil {
 		f.Close()
+		if err == syscall.EWOULDBLOCK {
+			err = ErrAlreadyLocked
+		}
 		return nil, err
 	}
 
@@ -73,3 +72,21 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 
 	return &LockedFile{File: f}, nil
 }
+
+// TryLockedOpenFile - tries a new write lock, functionality
+// it is similar to LockedOpenFile with with syscall.LOCK_EX
+// mode but along with syscall.LOCK_NB such that the function
+// doesn't wait forever but instead returns if it cannot
+// acquire a write lock.
+func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, syscall.LOCK_NB)
+}
+
+// LockedOpenFile - initializes a new lock and protects
+// the file from concurrent access across mount points.
+// This implementation doesn't support all the open
+// flags and shouldn't be considered as replacement
+// for os.OpenFile().
+func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, 0)
+}

pkg/lock/lock_solaris.go

@@ -24,17 +24,8 @@ import (
 	"syscall"
 )
 
-// LockedOpenFile - initializes a new lock and protects
-// the file from concurrent access across mount points.
-// This implementation doesn't support all the open
-// flags and shouldn't be considered as replacement
-// for os.OpenFile().
-func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
-	var lock syscall.Flock_t
-	lock.Start = 0
-	lock.Len = 0
-	lock.Pid = 0
-
+// lockedOpenFile is an internal function.
+func lockedOpenFile(path string, flag int, perm os.FileMode, rlockType int) (*LockedFile, error) {
 	var lockType int16
 	switch flag {
 	case syscall.O_RDONLY:
@@ -51,16 +42,24 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 		return nil, fmt.Errorf("Unsupported flag (%d)", flag)
 	}
 
-	lock.Type = lockType
-	lock.Whence = 0
+	var lock = syscall.Flock_t{
+		Start:  0,
+		Len:    0,
+		Pid:    0,
+		Type:   lockType,
+		Whence: 0,
+	}
 
 	f, err := os.OpenFile(path, flag, perm)
 	if err != nil {
 		return nil, err
 	}
 
-	if err = syscall.FcntlFlock(f.Fd(), syscall.F_SETLKW, &lock); err != nil {
+	if err = syscall.FcntlFlock(f.Fd(), rlockType, &lock); err != nil {
 		f.Close()
+		if err == syscall.EAGAIN {
+			err = ErrLocked
+		}
 		return nil, err
 	}
 
@@ -81,3 +80,21 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 
 	return &LockedFile{f}, nil
 }
+
+// TryLockedOpenFile - tries a new write lock, functionality
+// it is similar to LockedOpenFile with with syscall.LOCK_EX
+// mode but along with syscall.LOCK_NB such that the function
+// doesn't wait forever but instead returns if it cannot
+// acquire a write lock.
+func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, syscall.F_SETLK)
+}
+
+// LockedOpenFile - initializes a new lock and protects
+// the file from concurrent access across mount points.
+// This implementation doesn't support all the open
+// flags and shouldn't be considered as replacement
+// for os.OpenFile().
+func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, syscall.F_SETLKW)
+}

pkg/lock/lock_windows.go

@@ -19,7 +19,6 @@
 package lock
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"syscall"
@@ -31,24 +30,25 @@ import (
 var (
 	modkernel32    = syscall.NewLazyDLL("kernel32.dll")
 	procLockFileEx = modkernel32.NewProc("LockFileEx")
-
-	errLocked = errors.New("The process cannot access the file because another process has locked a portion of the file.")
 )
 
 const (
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
+	lockFileExclusiveLock   = 2
+	lockFileFailImmediately = 1
+
 	// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
 	errLockViolation syscall.Errno = 0x21
 )
 
-// LockedOpenFile - initializes a new lock and protects
-// the file from concurrent access.
-func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+// lockedOpenFile is an internal function.
+func lockedOpenFile(path string, flag int, perm os.FileMode, lockType uint32) (*LockedFile, error) {
 	f, err := open(path, flag, perm)
 	if err != nil {
 		return nil, err
 	}
 
-	if err = lockFile(syscall.Handle(f.Fd()), 0); err != nil {
+	if err = lockFile(syscall.Handle(f.Fd()), lockType); err != nil {
 		f.Close()
 		return nil, err
 	}
@@ -71,6 +71,21 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 	return &LockedFile{File: f}, nil
 }
 
+// TryLockedOpenFile - tries a new write lock, functionality
+// it is similar to LockedOpenFile with with syscall.LOCK_EX
+// mode but along with syscall.LOCK_NB such that the function
+// doesn't wait forever but instead returns if it cannot
+// acquire a write lock.
+func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, lockFileFailImmediately)
+}
+
+// LockedOpenFile - initializes a new lock and protects
+// the file from concurrent access.
+func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
+	return lockedOpenFile(path, flag, perm, 0)
+}
+
 // perm param is ignored, on windows file perms/NT acls
 // are not octet combinations. Providing access to NT
 // acls is out of scope here.
@@ -121,7 +136,7 @@ func open(path string, flag int, perm os.FileMode) (*os.File, error) {
 
 func lockFile(fd syscall.Handle, flags uint32) error {
 	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
-	var flag uint32 = 2 // Lockfile exlusive.
+	var flag uint32 = lockFileExclusiveLock // Lockfile exlusive.
 	flag |= flags
 
 	if fd == syscall.InvalidHandle {
@@ -131,8 +146,8 @@ func lockFile(fd syscall.Handle, flags uint32) error {
 	err := lockFileEx(fd, flag, 1, 0, &syscall.Overlapped{})
 	if err == nil {
 		return nil
-	} else if err.Error() == errLocked.Error() {
-		return errors.New("lock already acquired")
+	} else if err.Error() == "The process cannot access the file because another process has locked a portion of the file." {
+		return ErrAlreadyLocked
 	} else if err != errLockViolation {
 		return err
 	}