Lock free multipart backend implementation for FS (#5401)

pkg/ioutil/append-file_nix.go

@@ -0,0 +1,43 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ioutil
+
+import (
+	"io"
+	"os"
+)
+
+// AppendFile - appends the file "src" to the file "dst"
+func AppendFile(dst string, src string) error {
+	appendFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	defer appendFile.Close()
+
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+	// Allocate staging buffer.
+	var buf = make([]byte, defaultAppendBufferSize)
+	_, err = io.CopyBuffer(appendFile, srcFile, buf)
+	return err
+}

pkg/ioutil/append-file_windows.go

@@ -0,0 +1,43 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ioutil
+
+import (
+	"io"
+	"os"
+
+	"github.com/minio/minio/pkg/lock"
+)
+
+// AppendFile - appends the file "src" to the file "dst"
+func AppendFile(dst string, src string) error {
+	appendFile, err := lock.Open(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	defer appendFile.Close()
+
+	srcFile, err := lock.Open(src, os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+	// Allocate staging buffer.
+	var buf = make([]byte, defaultAppendBufferSize)
+	_, err = io.CopyBuffer(appendFile, srcFile, buf)
+	return err
+}

pkg/ioutil/ioutil.go

@@ -20,8 +20,13 @@ package ioutil
 
 import (
 	"io"
+
+	humanize "github.com/dustin/go-humanize"
 )
 
+// defaultAppendBufferSize - Default buffer size for the AppendFile
+const defaultAppendBufferSize = humanize.MiByte
+
 // WriteOnCloser implements io.WriteCloser and always
 // exectues at least one write operation if it is closed.
 //

pkg/ioutil/ioutil_test.go

@@ -18,6 +18,7 @@ package ioutil
 
 import (
 	goioutil "io/ioutil"
+	"os"
 	"testing"
 )
 
@@ -37,3 +38,38 @@ func TestCloseOnWriter(t *testing.T) {
 		t.Error("WriteOnCloser must be marked as HasWritten")
 	}
 }
+
+// Test for AppendFile.
+func TestAppendFile(t *testing.T) {
+	f, err := goioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	name1 := f.Name()
+	defer os.Remove(name1)
+	f.WriteString("aaaaaaaaaa")
+	f.Close()
+
+	f, err = goioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	name2 := f.Name()
+	defer os.Remove(name2)
+	f.WriteString("bbbbbbbbbb")
+	f.Close()
+
+	if err = AppendFile(name1, name2); err != nil {
+		t.Error(err)
+	}
+
+	b, err := goioutil.ReadFile(name1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := "aaaaaaaaaabbbbbbbbbb"
+	if string(b) != expected {
+		t.Errorf("AppendFile() failed, expected: %s, got %s", expected, string(b))
+	}
+}

pkg/lock/lock_nix.go

@@ -90,3 +90,8 @@ func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, er
 func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
 	return lockedOpenFile(path, flag, perm, 0)
 }
+
+// Open - Call os.OpenFile
+func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(path, flag, perm)
+}

pkg/lock/lock_windows.go

@@ -42,7 +42,7 @@ const (
 
 // lockedOpenFile is an internal function.
 func lockedOpenFile(path string, flag int, perm os.FileMode, lockType uint32) (*LockedFile, error) {
-	f, err := open(path, flag, perm)
+	f, err := Open(path, flag, perm)
 	if err != nil {
 		return nil, err
 	}
@@ -168,7 +168,7 @@ func fixLongPath(path string) string {
 // perm param is ignored, on windows file perms/NT acls
 // are not octet combinations. Providing access to NT
 // acls is out of scope here.
-func open(path string, flag int, perm os.FileMode) (*os.File, error) {
+func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
 	if path == "" {
 		return nil, syscall.ERROR_FILE_NOT_FOUND
 	}
@@ -190,6 +190,8 @@ func open(path string, flag int, perm os.FileMode) (*os.File, error) {
 		fallthrough
 	case syscall.O_WRONLY | syscall.O_CREAT:
 		access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
+	case syscall.O_WRONLY | syscall.O_CREAT | syscall.O_APPEND:
+		access = syscall.FILE_APPEND_DATA
 	default:
 		return nil, fmt.Errorf("Unsupported flag (%d)", flag)
 	}