rename all remaining packages to internal/ (#12418)

This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
2021-06-01 14:59:40 -07:00
committed by GitHub
parent bf87c4b1e4
commit 1f262daf6f
540 changed files with 757 additions and 778 deletions

102
internal/lock/lock.go Normal file
View File

@@ -0,0 +1,102 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package lock - implements filesystem locking wrappers around an
// open file descriptor.
package lock
import (
"errors"
"os"
"sync"
)
var (
// ErrAlreadyLocked is returned if the underlying fd is already locked.
ErrAlreadyLocked = errors.New("file already locked")
)
// RLockedFile represents a read locked file, implements a special
// closer which only closes the associated *os.File when the ref count.
// has reached zero, i.e when all the readers have given up their locks.
type RLockedFile struct {
*LockedFile
mutex sync.Mutex
refs int // Holds read lock refs.
}
// IsClosed - Check if the rlocked file is already closed.
func (r *RLockedFile) IsClosed() bool {
r.mutex.Lock()
defer r.mutex.Unlock()
return r.refs == 0
}
// IncLockRef - is used by called to indicate lock refs.
func (r *RLockedFile) IncLockRef() {
r.mutex.Lock()
r.refs++
r.mutex.Unlock()
}
// Close - this closer implements a special closer
// closes the underlying fd only when the refs
// reach zero.
func (r *RLockedFile) Close() (err error) {
r.mutex.Lock()
defer r.mutex.Unlock()
if r.refs == 0 {
return os.ErrInvalid
}
r.refs--
if r.refs == 0 {
err = r.File.Close()
}
return err
}
// Provides a new initialized read locked struct from *os.File
func newRLockedFile(lkFile *LockedFile) (*RLockedFile, error) {
if lkFile == nil {
return nil, os.ErrInvalid
}
return &RLockedFile{
LockedFile: lkFile,
refs: 1,
}, nil
}
// RLockedOpenFile - returns a wrapped read locked file, if the file
// doesn't exist at path returns an error.
func RLockedOpenFile(path string) (*RLockedFile, error) {
lkFile, err := LockedOpenFile(path, os.O_RDONLY, 0666)
if err != nil {
return nil, err
}
return newRLockedFile(lkFile)
}
// LockedFile represents a locked file
type LockedFile struct {
*os.File
}

101
internal/lock/lock_nix.go Normal file
View File

@@ -0,0 +1,101 @@
// +build !windows,!plan9,!solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lock
import (
"os"
"syscall"
)
// Internal function implements support for both
// blocking and non blocking lock type.
func lockedOpenFile(path string, flag int, perm os.FileMode, lockType int) (*LockedFile, error) {
switch flag {
case syscall.O_RDONLY:
lockType |= syscall.LOCK_SH
case syscall.O_WRONLY:
fallthrough
case syscall.O_RDWR:
fallthrough
case syscall.O_WRONLY | syscall.O_CREAT:
fallthrough
case syscall.O_RDWR | syscall.O_CREAT:
lockType |= syscall.LOCK_EX
default:
return nil, &os.PathError{
Op: "open",
Path: path,
Err: syscall.EINVAL,
}
}
f, err := os.OpenFile(path, flag|syscall.O_SYNC, perm)
if err != nil {
return nil, err
}
if err = syscall.Flock(int(f.Fd()), lockType); err != nil {
f.Close()
if err == syscall.EWOULDBLOCK {
err = ErrAlreadyLocked
}
return nil, err
}
st, err := os.Stat(path)
if err != nil {
f.Close()
return nil, err
}
if st.IsDir() {
f.Close()
return nil, &os.PathError{
Op: "open",
Path: path,
Err: syscall.EISDIR,
}
}
return &LockedFile{File: f}, nil
}
// TryLockedOpenFile - tries a new write lock, functionality
// it is similar to LockedOpenFile with with syscall.LOCK_EX
// mode but along with syscall.LOCK_NB such that the function
// doesn't wait forever but instead returns if it cannot
// acquire a write lock.
func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
return lockedOpenFile(path, flag, perm, syscall.LOCK_NB)
}
// LockedOpenFile - initializes a new lock and protects
// the file from concurrent access across mount points.
// This implementation doesn't support all the open
// flags and shouldn't be considered as replacement
// for os.OpenFile().
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
return lockedOpenFile(path, flag, perm, 0)
}
// Open - Call os.OpenFile
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(path, flag, perm)
}

View File

@@ -0,0 +1,109 @@
// +build solaris
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lock
import (
"os"
"syscall"
)
// lockedOpenFile is an internal function.
func lockedOpenFile(path string, flag int, perm os.FileMode, rlockType int) (*LockedFile, error) {
var lockType int16
switch flag {
case syscall.O_RDONLY:
lockType = syscall.F_RDLCK
case syscall.O_WRONLY:
fallthrough
case syscall.O_RDWR:
fallthrough
case syscall.O_WRONLY | syscall.O_CREAT:
fallthrough
case syscall.O_RDWR | syscall.O_CREAT:
lockType = syscall.F_WRLCK
default:
return nil, &os.PathError{
Op: "open",
Path: path,
Err: syscall.EINVAL,
}
}
var lock = syscall.Flock_t{
Start: 0,
Len: 0,
Pid: 0,
Type: lockType,
Whence: 0,
}
f, err := os.OpenFile(path, flag, perm)
if err != nil {
return nil, err
}
if err = syscall.FcntlFlock(f.Fd(), rlockType, &lock); err != nil {
f.Close()
if err == syscall.EAGAIN {
err = ErrAlreadyLocked
}
return nil, err
}
st, err := os.Stat(path)
if err != nil {
f.Close()
return nil, err
}
if st.IsDir() {
f.Close()
return nil, &os.PathError{
Op: "open",
Path: path,
Err: syscall.EISDIR,
}
}
return &LockedFile{f}, nil
}
// TryLockedOpenFile - tries a new write lock, functionality
// it is similar to LockedOpenFile with with syscall.LOCK_EX
// mode but along with syscall.LOCK_NB such that the function
// doesn't wait forever but instead returns if it cannot
// acquire a write lock.
func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
return lockedOpenFile(path, flag, perm, syscall.F_SETLK)
}
// LockedOpenFile - initializes a new lock and protects
// the file from concurrent access across mount points.
// This implementation doesn't support all the open
// flags and shouldn't be considered as replacement
// for os.OpenFile().
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
return lockedOpenFile(path, flag, perm, syscall.F_SETLKW)
}
// Open - Call os.OpenFile
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(path, flag, perm)
}

192
internal/lock/lock_test.go Normal file
View File

@@ -0,0 +1,192 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lock
import (
"io/ioutil"
"os"
"testing"
"time"
)
// Test lock fails.
func TestLockFail(t *testing.T) {
f, err := ioutil.TempFile("", "lock")
if err != nil {
t.Fatal(err)
}
f.Close()
defer func() {
err = os.Remove(f.Name())
if err != nil {
t.Fatal(err)
}
}()
_, err = LockedOpenFile(f.Name(), os.O_APPEND, 0600)
if err == nil {
t.Fatal("Should fail here")
}
}
// Tests lock directory fail.
func TestLockDirFail(t *testing.T) {
d, err := ioutil.TempDir("", "lockDir")
if err != nil {
t.Fatal(err)
}
defer func() {
err = os.Remove(d)
if err != nil {
t.Fatal(err)
}
}()
_, err = LockedOpenFile(d, os.O_APPEND, 0600)
if err == nil {
t.Fatal("Should fail here")
}
}
// Tests rwlock methods.
func TestRWLockedFile(t *testing.T) {
f, err := ioutil.TempFile("", "lock")
if err != nil {
t.Fatal(err)
}
f.Close()
defer func() {
err = os.Remove(f.Name())
if err != nil {
t.Fatal(err)
}
}()
rlk, err := RLockedOpenFile(f.Name())
if err != nil {
t.Fatal(err)
}
isClosed := rlk.IsClosed()
if isClosed {
t.Fatal("File ref count shouldn't be zero")
}
// Increase reference count to 2.
rlk.IncLockRef()
isClosed = rlk.IsClosed()
if isClosed {
t.Fatal("File ref count shouldn't be zero")
}
// Decrease reference count by 1.
if err = rlk.Close(); err != nil {
t.Fatal(err)
}
isClosed = rlk.IsClosed()
if isClosed {
t.Fatal("File ref count shouldn't be zero")
}
// Decrease reference count by 1.
if err = rlk.Close(); err != nil {
t.Fatal(err)
}
// Now file should be closed.
isClosed = rlk.IsClosed()
if !isClosed {
t.Fatal("File ref count should be zero")
}
// Closing a file again should result in invalid argument.
if err = rlk.Close(); err != os.ErrInvalid {
t.Fatal(err)
}
_, err = newRLockedFile(nil)
if err != os.ErrInvalid {
t.Fatal("Unexpected error", err)
}
}
// Tests lock and unlock semantics.
func TestLockAndUnlock(t *testing.T) {
f, err := ioutil.TempFile("", "lock")
if err != nil {
t.Fatal(err)
}
f.Close()
defer func() {
err = os.Remove(f.Name())
if err != nil {
t.Fatal(err)
}
}()
// lock the file
l, err := LockedOpenFile(f.Name(), os.O_WRONLY, 0600)
if err != nil {
t.Fatal(err)
}
// unlock the file
if err = l.Close(); err != nil {
t.Fatal(err)
}
// try lock the unlocked file
dupl, err := LockedOpenFile(f.Name(), os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
t.Errorf("err = %v, want %v", err, nil)
}
// blocking on locked file
locked := make(chan struct{}, 1)
go func() {
bl, blerr := LockedOpenFile(f.Name(), os.O_WRONLY, 0600)
if blerr != nil {
t.Error(blerr)
return
}
locked <- struct{}{}
if blerr = bl.Close(); blerr != nil {
t.Error(blerr)
return
}
}()
select {
case <-locked:
t.Error("unexpected unblocking")
case <-time.After(100 * time.Millisecond):
}
// unlock
if err = dupl.Close(); err != nil {
t.Fatal(err)
}
// the previously blocked routine should be unblocked
select {
case <-locked:
case <-time.After(1 * time.Second):
t.Error("unexpected blocking")
}
}

View File

@@ -0,0 +1,264 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lock
import (
"fmt"
"os"
"path/filepath"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var (
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procLockFileEx = modkernel32.NewProc("LockFileEx")
)
const (
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
lockFileExclusiveLock = 2
lockFileFailImmediately = 1
// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
errLockViolation syscall.Errno = 0x21
)
// lockedOpenFile is an internal function.
func lockedOpenFile(path string, flag int, perm os.FileMode, lockType uint32) (*LockedFile, error) {
f, err := Open(path, flag, perm)
if err != nil {
return nil, err
}
if err = lockFile(syscall.Handle(f.Fd()), lockType); err != nil {
f.Close()
return nil, err
}
st, err := os.Stat(path)
if err != nil {
f.Close()
return nil, err
}
if st.IsDir() {
f.Close()
return nil, &os.PathError{
Op: "open",
Path: path,
Err: syscall.EISDIR,
}
}
return &LockedFile{File: f}, nil
}
// TryLockedOpenFile - tries a new write lock, functionality
// it is similar to LockedOpenFile with with syscall.LOCK_EX
// mode but along with syscall.LOCK_NB such that the function
// doesn't wait forever but instead returns if it cannot
// acquire a write lock.
func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
var lockType uint32 = lockFileFailImmediately | lockFileExclusiveLock
switch flag {
case syscall.O_RDONLY:
// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-lockfileex
//lint:ignore SA4016 Reasons
lockType = lockFileFailImmediately | 0 // Set this to enable shared lock and fail immediately.
}
return lockedOpenFile(path, flag, perm, lockType)
}
// LockedOpenFile - initializes a new lock and protects
// the file from concurrent access.
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
var lockType uint32 = lockFileExclusiveLock
switch flag {
case syscall.O_RDONLY:
// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-lockfileex
lockType = 0 // Set this to enable shared lock.
}
return lockedOpenFile(path, flag, perm, lockType)
}
// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
func fixLongPath(path string) string {
// Do nothing (and don't allocate) if the path is "short".
// Empirically (at least on the Windows Server 2013 builder),
// the kernel is arbitrarily okay with < 248 bytes. That
// matches what the docs above say:
// "When using an API to create a directory, the specified
// path cannot be so long that you cannot append an 8.3 file
// name (that is, the directory name cannot exceed MAX_PATH
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
//
// The MSDN docs appear to say that a normal path that is 248 bytes long
// will work; empirically the path must be less then 248 bytes long.
if len(path) < 248 {
// Don't fix. (This is how Go 1.7 and earlier worked,
// not automatically generating the \\?\ form)
return path
}
// The extended form begins with \\?\, as in
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
// The extended form disables evaluation of . and .. path
// elements and disables the interpretation of / as equivalent
// to \. The conversion here rewrites / to \ and elides
// . elements as well as trailing or duplicate separators. For
// simplicity it avoids the conversion entirely for relative
// paths or paths containing .. elements. For now,
// \\server\share paths are not converted to
// \\?\UNC\server\share paths because the rules for doing so
// are less well-specified.
if len(path) >= 2 && path[:2] == `\\` {
// Don't canonicalize UNC paths.
return path
}
if !filepath.IsAbs(path) {
// Relative path
return path
}
const prefix = `\\?`
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
copy(pathbuf, prefix)
n := len(path)
r, w := 0, len(prefix)
for r < n {
switch {
case os.IsPathSeparator(path[r]):
// empty block
r++
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
// /./
r++
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
// /../ is currently unhandled
return path
default:
pathbuf[w] = '\\'
w++
for ; r < n && !os.IsPathSeparator(path[r]); r++ {
pathbuf[w] = path[r]
w++
}
}
}
// A drive's root directory needs a trailing \
if w == len(`\\?\c:`) {
pathbuf[w] = '\\'
w++
}
return string(pathbuf[:w])
}
// Open - perm param is ignored, on windows file perms/NT acls
// are not octet combinations. Providing access to NT
// acls is out of scope here.
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
if path == "" {
return nil, syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(fixLongPath(path))
if err != nil {
return nil, err
}
var access uint32
switch flag {
case syscall.O_RDONLY:
access = syscall.GENERIC_READ
case syscall.O_WRONLY:
access = syscall.GENERIC_WRITE
case syscall.O_RDWR:
fallthrough
case syscall.O_RDWR | syscall.O_CREAT:
fallthrough
case syscall.O_WRONLY | syscall.O_CREAT:
access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
case syscall.O_WRONLY | syscall.O_CREAT | syscall.O_APPEND:
access = syscall.FILE_APPEND_DATA
default:
return nil, fmt.Errorf("Unsupported flag (%d)", flag)
}
var createflag uint32
switch {
case flag&syscall.O_CREAT == syscall.O_CREAT:
createflag = syscall.OPEN_ALWAYS
default:
createflag = syscall.OPEN_EXISTING
}
shareflag := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
accessAttr := uint32(syscall.FILE_ATTRIBUTE_NORMAL | 0x80000000)
fd, err := syscall.CreateFile(pathp, access, shareflag, nil, createflag, accessAttr, 0)
if err != nil {
return nil, err
}
return os.NewFile(uintptr(fd), path), nil
}
func lockFile(fd syscall.Handle, flags uint32) error {
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
if fd == syscall.InvalidHandle {
return nil
}
err := lockFileEx(fd, flags, 1, 0, &syscall.Overlapped{})
if err == nil {
return nil
} else if err.Error() == "The process cannot access the file because another process has locked a portion of the file." {
return ErrAlreadyLocked
} else if err != errLockViolation {
return err
}
return nil
}
func lockFileEx(h syscall.Handle, flags, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
var reserved = uint32(0)
r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags),
uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return
}

View File

@@ -0,0 +1,60 @@
// +build windows
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lock
import (
"strings"
"testing"
)
func TestFixLongPath(t *testing.T) {
// 248 is long enough to trigger the longer-than-248 checks in
// fixLongPath, but short enough not to make a path component
// longer than 255, which is illegal on Windows. (which
// doesn't really matter anyway, since this is purely a string
// function we're testing, and it's not actually being used to
// do a system call)
veryLong := "l" + strings.Repeat("o", 248) + "ng"
for _, test := range []struct{ in, want string }{
// Short; unchanged:
{`C:\short.txt`, `C:\short.txt`},
{`C:\`, `C:\`},
{`C:`, `C:`},
// The "long" substring is replaced by a looooooong
// string which triggers the rewriting. Except in the
// cases below where it doesn't.
{`C:\long\foo.txt`, `\\?\C:\long\foo.txt`},
{`C:/long/foo.txt`, `\\?\C:\long\foo.txt`},
{`C:\long\foo\\bar\.\baz\\`, `\\?\C:\long\foo\bar\baz`},
{`\\unc\path`, `\\unc\path`},
{`long.txt`, `long.txt`},
{`C:long.txt`, `C:long.txt`},
{`c:\long\..\bar\baz`, `c:\long\..\bar\baz`},
{`\\?\c:\long\foo.txt`, `\\?\c:\long\foo.txt`},
{`\\?\c:\long/foo.txt`, `\\?\c:\long/foo.txt`},
} {
in := strings.Replace(test.in, "long", veryLong, -1)
want := strings.Replace(test.want, "long", veryLong, -1)
if got := fixLongPath(in); got != want {
got = strings.Replace(got, veryLong, "long", -1)
t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want)
}
}
}