posix: Deprecate custom removeAll/mkdirAll implementations. (#4808)

Since go1.8 os.RemoveAll and os.MkdirAll both support long
path names i.e UNC path on windows. The code we are carrying
was directly borrowed from `pkg/os` package and doesn't need
to be in our repo anymore. As a side affect this also
addresses our codecoverage issue.

Refer #4658

pkg/lock/lock_windows.go

@@ -21,6 +21,7 @@ package lock
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"syscall"
 	"unsafe"
 
@@ -86,6 +87,86 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 	return lockedOpenFile(path, flag, perm, 0)
 }
 
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less then 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !filepath.IsAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}
+
 // perm param is ignored, on windows file perms/NT acls
 // are not octet combinations. Providing access to NT
 // acls is out of scope here.
@@ -94,7 +175,7 @@ func open(path string, flag int, perm os.FileMode) (*os.File, error) {
 		return nil, syscall.ERROR_FILE_NOT_FOUND
 	}
 
-	pathp, err := syscall.UTF16PtrFromString(path)
+	pathp, err := syscall.UTF16PtrFromString(fixLongPath(path))
 	if err != nil {
 		return nil, err
 	}

pkg/lock/lock_windows_test.go

@@ -0,0 +1,59 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lock
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFixLongPath(t *testing.T) {
+	// 248 is long enough to trigger the longer-than-248 checks in
+	// fixLongPath, but short enough not to make a path component
+	// longer than 255, which is illegal on Windows. (which
+	// doesn't really matter anyway, since this is purely a string
+	// function we're testing, and it's not actually being used to
+	// do a system call)
+	veryLong := "l" + strings.Repeat("o", 248) + "ng"
+	for _, test := range []struct{ in, want string }{
+		// Short; unchanged:
+		{`C:\short.txt`, `C:\short.txt`},
+		{`C:\`, `C:\`},
+		{`C:`, `C:`},
+		// The "long" substring is replaced by a looooooong
+		// string which triggers the rewriting. Except in the
+		// cases below where it doesn't.
+		{`C:\long\foo.txt`, `\\?\C:\long\foo.txt`},
+		{`C:/long/foo.txt`, `\\?\C:\long\foo.txt`},
+		{`C:\long\foo\\bar\.\baz\\`, `\\?\C:\long\foo\bar\baz`},
+		{`\\unc\path`, `\\unc\path`},
+		{`long.txt`, `long.txt`},
+		{`C:long.txt`, `C:long.txt`},
+		{`c:\long\..\bar\baz`, `c:\long\..\bar\baz`},
+		{`\\?\c:\long\foo.txt`, `\\?\c:\long\foo.txt`},
+		{`\\?\c:\long/foo.txt`, `\\?\c:\long/foo.txt`},
+	} {
+		in := strings.Replace(test.in, "long", veryLong, -1)
+		want := strings.Replace(test.want, "long", veryLong, -1)
+		if got := fixLongPath(in); got != want {
+			got = strings.Replace(got, veryLong, "long", -1)
+			t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want)
+		}
+	}
+}

pkg/x/os/stat_windows.go

@@ -37,6 +37,86 @@ import (
 
 const errSharingViolation syscall.Errno = 32
 
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less then 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !filepath.IsAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os1.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os1.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os1.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os1.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}
+
 // Stat returns a FileInfo structure describing the
 // named file. If there is an error, it will be of type
 // *PathError.
@@ -51,7 +131,7 @@ func Stat(name string) (os1.FileInfo, error) {
 	if name == os1.DevNull {
 		return &devNullStat, nil
 	}
-	namep, err := syscall.UTF16PtrFromString(name)
+	namep, err := syscall.UTF16PtrFromString(fixLongPath(name))
 	if err != nil {
 		return nil, &os1.PathError{Op: "Stat", Path: name, Err: err}
 	}

pkg/x/os/stat_windows_test.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	os1 "os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 	"unsafe"
@@ -207,3 +208,38 @@ func TestStatSymlinkLoop(t *testing.T) {
 		}
 	}
 }
+
+func TestFixLongPath(t *testing.T) {
+	// 248 is long enough to trigger the longer-than-248 checks in
+	// fixLongPath, but short enough not to make a path component
+	// longer than 255, which is illegal on Windows. (which
+	// doesn't really matter anyway, since this is purely a string
+	// function we're testing, and it's not actually being used to
+	// do a system call)
+	veryLong := "l" + strings.Repeat("o", 248) + "ng"
+	for _, test := range []struct{ in, want string }{
+		// Short; unchanged:
+		{`C:\short.txt`, `C:\short.txt`},
+		{`C:\`, `C:\`},
+		{`C:`, `C:`},
+		// The "long" substring is replaced by a looooooong
+		// string which triggers the rewriting. Except in the
+		// cases below where it doesn't.
+		{`C:\long\foo.txt`, `\\?\C:\long\foo.txt`},
+		{`C:/long/foo.txt`, `\\?\C:\long\foo.txt`},
+		{`C:\long\foo\\bar\.\baz\\`, `\\?\C:\long\foo\bar\baz`},
+		{`\\unc\path`, `\\unc\path`},
+		{`long.txt`, `long.txt`},
+		{`C:long.txt`, `C:long.txt`},
+		{`c:\long\..\bar\baz`, `c:\long\..\bar\baz`},
+		{`\\?\c:\long\foo.txt`, `\\?\c:\long\foo.txt`},
+		{`\\?\c:\long/foo.txt`, `\\?\c:\long/foo.txt`},
+	} {
+		in := strings.Replace(test.in, "long", veryLong, -1)
+		want := strings.Replace(test.want, "long", veryLong, -1)
+		if got := fixLongPath(in); got != want {
+			got = strings.Replace(got, veryLong, "long", -1)
+			t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want)
+		}
+	}
+}