mirror of
https://github.com/minio/minio.git
synced 2025-11-11 06:20:14 -05:00
fs: Re-implement object layer to remember the fd (#3509)
This patch re-writes FS backend to support shared backend sharing locks for safe concurrent access across multiple servers.
This commit is contained in:
Harshavardhana
GitHub
102
pkg/lock/lock.go
Normal file
102
pkg/lock/lock.go
Normal file
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Package lock - implements filesystem locking wrappers around an
|
||||
// open file descriptor.
|
||||
package lock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// RLockedFile represents a read locked file, implements a special
|
||||
// closer which only closes the associated *os.File when the ref count.
|
||||
// has reached zero, i.e when all the readers have given up their locks.
|
||||
type RLockedFile struct {
|
||||
*LockedFile
|
||||
mutex sync.Mutex
|
||||
refs int // Holds read lock refs.
|
||||
}
|
||||
|
||||
// IsClosed - Check if the rlocked file is already closed.
|
||||
func (r *RLockedFile) IsClosed() bool {
|
||||
r.mutex.Lock()
|
||||
defer r.mutex.Unlock()
|
||||
return r.refs == 0
|
||||
}
|
||||
|
||||
// IncLockRef - is used by called to indicate lock refs.
|
||||
func (r *RLockedFile) IncLockRef() {
|
||||
r.mutex.Lock()
|
||||
r.refs++
|
||||
r.mutex.Unlock()
|
||||
}
|
||||
|
||||
// Close - this closer implements a special closer
|
||||
// closes the underlying fd only when the refs
|
||||
// reach zero.
|
||||
func (r *RLockedFile) Close() (err error) {
|
||||
r.mutex.Lock()
|
||||
defer r.mutex.Unlock()
|
||||
|
||||
if r.refs == 0 {
|
||||
return os.ErrInvalid
|
||||
}
|
||||
|
||||
r.refs--
|
||||
if r.refs == 0 {
|
||||
err = r.File.Close()
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Provides a new initialized read locked struct from *os.File
|
||||
func newRLockedFile(lkFile *LockedFile) (*RLockedFile, error) {
|
||||
if lkFile == nil {
|
||||
return nil, os.ErrInvalid
|
||||
}
|
||||
|
||||
return &RLockedFile{
|
||||
LockedFile: lkFile,
|
||||
refs: 1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// RLockedOpenFile - returns a wrapped read locked file, if the file
|
||||
// doesn't exist at path returns an error.
|
||||
func RLockedOpenFile(path string) (*RLockedFile, error) {
|
||||
lkFile, err := LockedOpenFile(path, os.O_RDONLY, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newRLockedFile(lkFile)
|
||||
|
||||
}
|
||||
|
||||
// LockedFile represents a locked file, implements a helper
|
||||
// method Size(), represents the size of the underlying object.
|
||||
type LockedFile struct {
|
||||
*os.File
|
||||
size int64
|
||||
}
|
||||
|
||||
// Size - size of the underlying locked file.
|
||||
func (l *LockedFile) Size() int64 {
|
||||
return l.size
|
||||
}
|
||||
75
pkg/lock/lock_nix.go
Normal file
75
pkg/lock/lock_nix.go
Normal file
@@ -0,0 +1,75 @@
|
||||
// +build !windows,!plan9,!solaris
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// LockedOpenFile - initializes a new lock and protects
|
||||
// the file from concurrent access across mount points.
|
||||
// This implementation doesn't support all the open
|
||||
// flags and shouldn't be considered as replacement
|
||||
// for os.OpenFile().
|
||||
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
|
||||
var lockType int
|
||||
switch flag {
|
||||
case syscall.O_RDONLY:
|
||||
lockType = syscall.LOCK_SH
|
||||
case syscall.O_WRONLY:
|
||||
fallthrough
|
||||
case syscall.O_RDWR:
|
||||
fallthrough
|
||||
case syscall.O_WRONLY | syscall.O_CREAT:
|
||||
fallthrough
|
||||
case syscall.O_RDWR | syscall.O_CREAT:
|
||||
lockType = syscall.LOCK_EX
|
||||
default:
|
||||
return nil, fmt.Errorf("Unsupported flag (%d)", flag)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(path, flag|syscall.O_SYNC, perm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = syscall.Flock(int(f.Fd()), lockType); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if st.IsDir() {
|
||||
f.Close()
|
||||
return nil, &os.PathError{
|
||||
Op: "open",
|
||||
Path: path,
|
||||
Err: syscall.EISDIR,
|
||||
}
|
||||
}
|
||||
|
||||
return &LockedFile{File: f, size: st.Size()}, nil
|
||||
}
|
||||
192
pkg/lock/lock_test.go
Normal file
192
pkg/lock/lock_test.go
Normal file
@@ -0,0 +1,192 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Test lock fails.
|
||||
func TestLockFail(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
defer func() {
|
||||
err = os.Remove(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = LockedOpenFile(f.Name(), os.O_APPEND, 0600)
|
||||
if err == nil {
|
||||
t.Fatal("Should fail here")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests lock directory fail.
|
||||
func TestLockDirFail(t *testing.T) {
|
||||
d, err := ioutil.TempDir("", "lockDir")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
err = os.Remove(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = LockedOpenFile(d, os.O_APPEND, 0600)
|
||||
if err == nil {
|
||||
t.Fatal("Should fail here")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests rwlock methods.
|
||||
func TestRWLockedFile(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
defer func() {
|
||||
err = os.Remove(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
rlk, err := RLockedOpenFile(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rlk.Size() != 0 {
|
||||
t.Fatal("File size should be zero", rlk.Size())
|
||||
}
|
||||
isClosed := rlk.IsClosed()
|
||||
if isClosed {
|
||||
t.Fatal("File ref count shouldn't be zero")
|
||||
}
|
||||
|
||||
// Increase reference count to 2.
|
||||
rlk.IncLockRef()
|
||||
|
||||
isClosed = rlk.IsClosed()
|
||||
if isClosed {
|
||||
t.Fatal("File ref count shouldn't be zero")
|
||||
}
|
||||
|
||||
// Decrease reference count by 1.
|
||||
if err = rlk.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
isClosed = rlk.IsClosed()
|
||||
if isClosed {
|
||||
t.Fatal("File ref count shouldn't be zero")
|
||||
}
|
||||
|
||||
// Decrease reference count by 1.
|
||||
if err = rlk.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Now file should be closed.
|
||||
isClosed = rlk.IsClosed()
|
||||
if !isClosed {
|
||||
t.Fatal("File ref count should be zero")
|
||||
}
|
||||
|
||||
// Closing a file again should result in invalid argument.
|
||||
if err = rlk.Close(); err != os.ErrInvalid {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = newRLockedFile(nil)
|
||||
if err != os.ErrInvalid {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests lock and unlock semantics.
|
||||
func TestLockAndUnlock(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
defer func() {
|
||||
err = os.Remove(f.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// lock the file
|
||||
l, err := LockedOpenFile(f.Name(), os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// unlock the file
|
||||
if err = l.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// try lock the unlocked file
|
||||
dupl, err := LockedOpenFile(f.Name(), os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Errorf("err = %v, want %v", err, nil)
|
||||
}
|
||||
|
||||
// blocking on locked file
|
||||
locked := make(chan struct{}, 1)
|
||||
go func() {
|
||||
bl, blerr := LockedOpenFile(f.Name(), os.O_WRONLY, 0600)
|
||||
if blerr != nil {
|
||||
t.Fatal(blerr)
|
||||
}
|
||||
locked <- struct{}{}
|
||||
if blerr = bl.Close(); blerr != nil {
|
||||
t.Fatal(blerr)
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-locked:
|
||||
t.Error("unexpected unblocking")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
// unlock
|
||||
if err = dupl.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// the previously blocked routine should be unblocked
|
||||
select {
|
||||
case <-locked:
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Error("unexpected blocking")
|
||||
}
|
||||
}
|
||||
172
pkg/lock/lock_windows.go
Normal file
172
pkg/lock/lock_windows.go
Normal file
@@ -0,0 +1,172 @@
|
||||
// +build windows
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package lock
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||
procLockFileEx = modkernel32.NewProc("LockFileEx")
|
||||
|
||||
errLocked = errors.New("The process cannot access the file because another process has locked a portion of the file.")
|
||||
)
|
||||
|
||||
const (
|
||||
// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
|
||||
errLockViolation syscall.Errno = 0x21
|
||||
)
|
||||
|
||||
// LockedOpenFile - initializes a new lock and protects
|
||||
// the file from concurrent access.
|
||||
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
|
||||
f, err := open(path, flag, perm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = lockFile(syscall.Handle(f.Fd()), 0); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if st.IsDir() {
|
||||
f.Close()
|
||||
return nil, &os.PathError{
|
||||
Op: "open",
|
||||
Path: path,
|
||||
Err: syscall.EISDIR,
|
||||
}
|
||||
}
|
||||
|
||||
return &LockedFile{File: f, size: st.Size()}, nil
|
||||
}
|
||||
|
||||
func makeInheritSa() *syscall.SecurityAttributes {
|
||||
var sa syscall.SecurityAttributes
|
||||
sa.Length = uint32(unsafe.Sizeof(sa))
|
||||
sa.InheritHandle = 1
|
||||
return &sa
|
||||
}
|
||||
|
||||
// perm param is ignored, on windows file perms/NT acls
|
||||
// are not octet combinations. Providing access to NT
|
||||
// acls is out of scope here.
|
||||
func open(path string, flag int, perm os.FileMode) (*os.File, error) {
|
||||
if path == "" {
|
||||
return nil, syscall.ERROR_FILE_NOT_FOUND
|
||||
}
|
||||
|
||||
pathp, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var access uint32
|
||||
switch flag {
|
||||
case syscall.O_RDONLY:
|
||||
access = syscall.GENERIC_READ
|
||||
case syscall.O_WRONLY:
|
||||
access = syscall.GENERIC_WRITE
|
||||
case syscall.O_RDWR:
|
||||
access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
|
||||
case syscall.O_RDWR | syscall.O_CREAT:
|
||||
access = syscall.GENERIC_ALL
|
||||
case syscall.O_WRONLY | syscall.O_CREAT:
|
||||
access = syscall.GENERIC_ALL
|
||||
}
|
||||
|
||||
if flag&syscall.O_APPEND != 0 {
|
||||
access &^= syscall.GENERIC_WRITE
|
||||
access |= syscall.FILE_APPEND_DATA
|
||||
}
|
||||
|
||||
var sa *syscall.SecurityAttributes
|
||||
if flag&syscall.O_CLOEXEC == 0 {
|
||||
sa = makeInheritSa()
|
||||
}
|
||||
|
||||
var createflag uint32
|
||||
switch {
|
||||
case flag&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL):
|
||||
createflag = syscall.CREATE_NEW
|
||||
case flag&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC):
|
||||
createflag = syscall.CREATE_ALWAYS
|
||||
case flag&syscall.O_CREAT == syscall.O_CREAT:
|
||||
createflag = syscall.OPEN_ALWAYS
|
||||
case flag&syscall.O_TRUNC == syscall.O_TRUNC:
|
||||
createflag = syscall.TRUNCATE_EXISTING
|
||||
default:
|
||||
createflag = syscall.OPEN_EXISTING
|
||||
}
|
||||
|
||||
shareflag := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
|
||||
accessAttr := uint32(syscall.FILE_ATTRIBUTE_NORMAL | 0x80000000)
|
||||
|
||||
fd, err := syscall.CreateFile(pathp, access, shareflag, sa, createflag, accessAttr, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return os.NewFile(uintptr(fd), path), nil
|
||||
}
|
||||
|
||||
func lockFile(fd syscall.Handle, flags uint32) error {
|
||||
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
|
||||
var flag uint32 = 2 // Lockfile exlusive.
|
||||
flag |= flags
|
||||
|
||||
if fd == syscall.InvalidHandle {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := lockFileEx(fd, flag, 1, 0, &syscall.Overlapped{})
|
||||
if err == nil {
|
||||
return nil
|
||||
} else if err.Error() == errLocked.Error() {
|
||||
return errors.New("lock already acquired")
|
||||
} else if err != errLockViolation {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockFileEx(h syscall.Handle, flags, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
|
||||
var reserved = uint32(0)
|
||||
r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
|
||||
if r1 == 0 {
|
||||
if e1 != 0 {
|
||||
err = error(e1)
|
||||
} else {
|
||||
err = syscall.EINVAL
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -298,7 +298,7 @@ func (d config) Diff(c Config) ([]structs.Field, error) {
|
||||
currFields := structs.Fields(d.Data())
|
||||
newFields := structs.Fields(c.Data())
|
||||
|
||||
found := false
|
||||
var found bool
|
||||
for _, currField := range currFields {
|
||||
found = false
|
||||
for _, newField := range newFields {
|
||||
@@ -324,7 +324,7 @@ func (d config) DeepDiff(c Config) ([]structs.Field, error) {
|
||||
currFields := structs.Fields(d.Data())
|
||||
newFields := structs.Fields(c.Data())
|
||||
|
||||
found := false
|
||||
var found bool
|
||||
for _, currField := range currFields {
|
||||
found = false
|
||||
for _, newField := range newFields {
|
||||
|
||||
Reference in New Issue
Block a user