windows: Support all REPARSE_POINT attrib files properly. (#4203)

This change adopts the upstream fix in this regard at
https://go-review.googlesource.com/#/c/41834/ for Minio's
purposes.

Go's current os.Stat() lacks support for lot of strange
windows files such as

 - share symlinks on SMB2
 - symlinks on docker nanoserver
 - de-duplicated files on NTFS de-duplicated volume.

This PR attempts to incorporate the change mentioned here

   https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/

The article suggests to use Windows I/O manager to
dereference the symbolic link.

Fixes #4122

pkg/disk/stat_windows.go

@@ -19,9 +19,10 @@
 package disk
 
 import (
-	"os"
 	"syscall"
 	"unsafe"
+
+	os2 "github.com/minio/minio/pkg/x/os"
 )
 
 var (
@@ -43,7 +44,7 @@ var (
 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
 func GetInfo(path string) (info Info, err error) {
 	// Stat to know if the path exists.
-	if _, err = os.Stat(path); err != nil {
+	if _, err = os2.Stat(path); err != nil {
 		return Info{}, err
 	}
 

pkg/lock/lock_windows.go

@@ -24,6 +24,8 @@ import (
 	"os"
 	"syscall"
 	"unsafe"
+
+	os2 "github.com/minio/minio/pkg/x/os"
 )
 
 var (
@@ -51,7 +53,7 @@ func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error
 		return nil, err
 	}
 
-	st, err := os.Stat(path)
+	st, err := os2.Stat(path)
 	if err != nil {
 		f.Close()
 		return nil, err
@@ -140,7 +142,8 @@ func lockFile(fd syscall.Handle, flags uint32) error {
 
 func lockFileEx(h syscall.Handle, flags, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
 	var reserved = uint32(0)
-	r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
+	r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags),
+		uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
 	if r1 == 0 {
 		if e1 != 0 {
 			err = error(e1)

pkg/quick/encoding.go

@@ -22,11 +22,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 
+	os2 "github.com/minio/minio/pkg/x/os"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -122,7 +122,7 @@ func saveFileConfig(filename string, v interface{}) error {
 // decoder format according to the filename extension. If no
 // extension is provided, json will be selected by default.
 func loadFileConfig(filename string, v interface{}) error {
-	if _, err := os.Stat(filename); err != nil {
+	if _, err := os2.Stat(filename); err != nil {
 		return err
 	}
 	fileData, err := ioutil.ReadFile(filename)

pkg/safe/safe_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Client (C) 2015 Minio, Inc.
+ * Minio Client (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import (
 	"path"
 	"testing"
 
+	os2 "github.com/minio/minio/pkg/x/os"
 	. "gopkg.in/check.v1"
 )
 
@@ -47,7 +48,7 @@ func (s *MySuite) TearDownSuite(c *C) {
 func (s *MySuite) TestSafeAbort(c *C) {
 	f, err := CreateFile(path.Join(s.root, "testfile-abort"))
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "testfile-abort"))
+	_, err = os2.Stat(path.Join(s.root, "testfile-abort"))
 	c.Assert(err, Not(IsNil))
 	err = f.Abort()
 	c.Assert(err, IsNil)
@@ -58,11 +59,11 @@ func (s *MySuite) TestSafeAbort(c *C) {
 func (s *MySuite) TestSafeClose(c *C) {
 	f, err := CreateFile(path.Join(s.root, "testfile-close"))
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "testfile-close"))
+	_, err = os2.Stat(path.Join(s.root, "testfile-close"))
 	c.Assert(err, Not(IsNil))
 	err = f.Close()
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "testfile-close"))
+	_, err = os2.Stat(path.Join(s.root, "testfile-close"))
 	c.Assert(err, IsNil)
 	err = os.Remove(path.Join(s.root, "testfile-close"))
 	c.Assert(err, IsNil)
@@ -73,7 +74,7 @@ func (s *MySuite) TestSafeClose(c *C) {
 func (s *MySuite) TestSafe(c *C) {
 	f, err := CreateFile(path.Join(s.root, "testfile-safe"))
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "testfile-safe"))
+	_, err = os2.Stat(path.Join(s.root, "testfile-safe"))
 	c.Assert(err, Not(IsNil))
 	err = f.Close()
 	c.Assert(err, IsNil)
@@ -81,7 +82,7 @@ func (s *MySuite) TestSafe(c *C) {
 	c.Assert(err.Error(), Equals, "write on closed file")
 	err = f.Close()
 	c.Assert(err.Error(), Equals, "close on closed file")
-	_, err = os.Stat(path.Join(s.root, "testfile-safe"))
+	_, err = os2.Stat(path.Join(s.root, "testfile-safe"))
 	c.Assert(err, IsNil)
 	err = os.Remove(path.Join(s.root, "testfile-safe"))
 	c.Assert(err, IsNil)
@@ -90,11 +91,11 @@ func (s *MySuite) TestSafe(c *C) {
 func (s *MySuite) TestSafeAbortWrite(c *C) {
 	f, err := CreateFile(path.Join(s.root, "purgefile-abort"))
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "purgefile-abort"))
+	_, err = os2.Stat(path.Join(s.root, "purgefile-abort"))
 	c.Assert(err, Not(IsNil))
 	err = f.Abort()
 	c.Assert(err, IsNil)
-	_, err = os.Stat(path.Join(s.root, "purgefile-abort"))
+	_, err = os2.Stat(path.Join(s.root, "purgefile-abort"))
 	c.Assert(err, Not(IsNil))
 	err = f.Abort()
 	c.Assert(err.Error(), Equals, "abort on aborted file")

pkg/x/os/stat_others.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package os
+
+import os1 "os"
+
+// Stat returns a FileInfo structure describing the
+// named file. If there is an error, it will be of type
+// *PathError.
+func Stat(name string) (os1.FileInfo, error) {
+	return os1.Stat(name)
+}

pkg/x/os/stat_windows.go

@@ -0,0 +1,188 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// FIXME: Once we have a go version released with the
+// following fix https://go-review.googlesource.com/#/c/41834/.
+// We should actively purge this block.
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package os implements extended safe functions
+// for stdlib "os".
+package os
+
+import (
+	os1 "os"
+	"path/filepath"
+	"syscall"
+	"time"
+)
+
+const errSharingViolation syscall.Errno = 32
+
+// Stat returns a FileInfo structure describing the
+// named file. If there is an error, it will be of type
+// *PathError.
+func Stat(name string) (os1.FileInfo, error) {
+	if len(name) == 0 {
+		return nil, &os1.PathError{
+			Op:   "Stat",
+			Path: name,
+			Err:  syscall.Errno(syscall.ERROR_PATH_NOT_FOUND),
+		}
+	}
+	if name == os1.DevNull {
+		return &devNullStat, nil
+	}
+	namep, err := syscall.UTF16PtrFromString(name)
+	if err != nil {
+		return nil, &os1.PathError{Op: "Stat", Path: name, Err: err}
+	}
+
+	// Use Windows I/O manager to dereference the symbolic link, as per
+	// https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/
+	h, err := syscall.CreateFile(namep, 0, 0, nil,
+		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
+	if err != nil {
+		if err == errSharingViolation {
+			// try FindFirstFile now that CreateFile failed
+			return statWithFindFirstFile(name, namep)
+		}
+		return nil, &os1.PathError{Op: "CreateFile", Path: name, Err: err}
+	}
+	defer syscall.CloseHandle(h)
+
+	var d syscall.ByHandleFileInformation
+	if err = syscall.GetFileInformationByHandle(h, &d); err != nil {
+		return nil, &os1.PathError{
+			Op:   "GetFileInformationByHandle",
+			Path: name,
+			Err:  err,
+		}
+	}
+	return &fileStat{
+		name: filepath.Base(name),
+		sys: syscall.Win32FileAttributeData{
+			FileAttributes: d.FileAttributes,
+			CreationTime:   d.CreationTime,
+			LastAccessTime: d.LastAccessTime,
+			LastWriteTime:  d.LastWriteTime,
+			FileSizeHigh:   d.FileSizeHigh,
+			FileSizeLow:    d.FileSizeLow,
+		},
+		vol:   d.VolumeSerialNumber,
+		idxhi: d.FileIndexHigh,
+		idxlo: d.FileIndexLow,
+		// fileStat.path is used by os1.SameFile to decide, if it needs
+		// to fetch vol, idxhi and idxlo. But these are already set,
+		// so set fileStat.path to "" to prevent os1.SameFile doing it again.
+		// Also do not set fileStat.filetype, because it is only used for
+		// console and stdin/stdout. But you cannot call os1.Stat for these.
+	}, nil
+}
+
+// statWithFindFirstFile is used by Stat to handle special case of stating
+// c:\pagefile.sys. We might discovered other files need similar treatment.
+func statWithFindFirstFile(name string, namep *uint16) (os1.FileInfo, error) {
+	var fd syscall.Win32finddata
+	h, err := syscall.FindFirstFile(namep, &fd)
+	if err != nil {
+		return nil, &os1.PathError{Op: "FindFirstFile", Path: name, Err: err}
+	}
+	syscall.FindClose(h)
+
+	fullpath := name
+	if !filepath.IsAbs(fullpath) {
+		fullpath, err = syscall.FullPath(fullpath)
+		if err != nil {
+			return nil, &os1.PathError{Op: "FullPath", Path: name, Err: err}
+		}
+	}
+	return &fileStat{
+		name: filepath.Base(name),
+		path: fullpath,
+		sys: syscall.Win32FileAttributeData{
+			FileAttributes: fd.FileAttributes,
+			CreationTime:   fd.CreationTime,
+			LastAccessTime: fd.LastAccessTime,
+			LastWriteTime:  fd.LastWriteTime,
+			FileSizeHigh:   fd.FileSizeHigh,
+			FileSizeLow:    fd.FileSizeLow,
+		},
+	}, nil
+}
+
+// A fileStat is the implementation of os1.FileInfo returned by stat.
+type fileStat struct {
+	name     string
+	sys      syscall.Win32FileAttributeData
+	filetype uint32 // what syscall.GetFileType returns
+
+	path  string
+	vol   uint32
+	idxhi uint32
+	idxlo uint32
+}
+
+func (fs *fileStat) Name() string { return fs.name }
+func (fs *fileStat) IsDir() bool  { return fs.Mode().IsDir() }
+
+func (fs *fileStat) Size() int64 {
+	return int64(fs.sys.FileSizeHigh)<<32 + int64(fs.sys.FileSizeLow)
+}
+
+// devNullStat is fileStat structure describing DevNull file ("NUL").
+var devNullStat = fileStat{
+	name:  os1.DevNull,
+	vol:   0,
+	idxhi: 0,
+	idxlo: 0,
+}
+
+func (fs *fileStat) Mode() (m os1.FileMode) {
+	if fs == &devNullStat {
+		return os1.ModeDevice | os1.ModeCharDevice | 0666
+	}
+	if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 {
+		m |= 0444
+	} else {
+		m |= 0666
+	}
+	if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 {
+		return m | os1.ModeSymlink
+	}
+	if fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
+		m |= os1.ModeDir | 0111
+	}
+	switch fs.filetype {
+	case syscall.FILE_TYPE_PIPE:
+		m |= os1.ModeNamedPipe
+	case syscall.FILE_TYPE_CHAR:
+		m |= os1.ModeCharDevice
+	}
+	return m
+}
+
+func (fs *fileStat) ModTime() time.Time {
+	return time.Unix(0, fs.sys.LastWriteTime.Nanoseconds())
+}
+
+// Sys returns syscall.Win32FileAttributeData for file fs.
+func (fs *fileStat) Sys() interface{} { return &fs.sys }

pkg/x/os/stat_windows_test.go

@@ -0,0 +1,209 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package os
+
+import (
+	"io/ioutil"
+	os1 "os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"unsafe"
+)
+
+var (
+	modnetapi32 = syscall.NewLazyDLL("netapi32.dll")
+
+	procNetShareAdd = modnetapi32.NewProc("NetShareAdd")
+	procNetShareDel = modnetapi32.NewProc("NetShareDel")
+)
+
+func netShareAdd(serverName *uint16, level uint32, buf *byte, parmErr *uint16) (neterr error) {
+	r0, _, _ := syscall.Syscall6(procNetShareAdd.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(level), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(parmErr)), 0, 0)
+	if r0 != 0 {
+		neterr = syscall.Errno(r0)
+	}
+	return
+}
+
+func netShareDel(serverName *uint16, netName *uint16, reserved uint32) (neterr error) {
+	r0, _, _ := syscall.Syscall(procNetShareDel.Addr(), 3, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(netName)), uintptr(reserved))
+	if r0 != 0 {
+		neterr = syscall.Errno(r0)
+	}
+	return
+}
+
+type shareInfo2 struct {
+	Netname     *uint16
+	Type        uint32
+	Remark      *uint16
+	Permissions uint32
+	MaxUses     uint32
+	CurrentUses uint32
+	Path        *uint16
+	Passwd      *uint16
+}
+
+func sameFile(fi1, fi2 os1.FileInfo) bool {
+	fii1, ok1 := fi1.(*fileStat)
+	fii2, ok2 := fi2.(*fileStat)
+	if !ok1 || !ok2 {
+		return false
+	}
+	return fii1.vol == fii2.vol && fii1.idxhi == fii2.idxhi && fii1.idxlo == fii2.idxlo
+}
+
+func TestNetworkSymbolicLink(t *testing.T) {
+	dir, err := ioutil.TempDir("", "TestNetworkSymbolicLink")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os1.RemoveAll(dir)
+
+	oldwd, err := os1.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os1.Chdir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os1.Chdir(oldwd)
+
+	shareName := "GoSymbolicLinkTestShare" // hope no conflicts.
+	sharePath := filepath.Join(dir, shareName)
+	testDir := "TestDir"
+
+	err = os1.MkdirAll(filepath.Join(sharePath, testDir), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wShareName, err := syscall.UTF16PtrFromString(shareName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wSharePath, err := syscall.UTF16PtrFromString(sharePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := shareInfo2{
+		Netname:     wShareName,
+		Type:        0x00, // STYPE_DISKTREE
+		Remark:      nil,
+		Permissions: 0,
+		MaxUses:     1,
+		CurrentUses: 0,
+		Path:        wSharePath,
+		Passwd:      nil,
+	}
+
+	err = netShareAdd(nil, 2, (*byte)(unsafe.Pointer(&p)), nil)
+	if err != nil {
+		if err == syscall.ERROR_ACCESS_DENIED {
+			t.Skip("you don't have enough privileges to add network share")
+		}
+		t.Fatal(err)
+	}
+	defer func() {
+		err := netShareDel(nil, wShareName, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	UNCPath := `\\localhost\` + shareName + `\`
+
+	fi1, err := Stat(sharePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fi2, err := Stat(UNCPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !sameFile(fi1, fi2) {
+		t.Fatalf("%q and %q should be the same directory, but not", sharePath, UNCPath)
+	}
+
+	target := filepath.Join(UNCPath, testDir)
+	link := "link"
+
+	err = os1.Symlink(target, link)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os1.Remove(link)
+
+	got, err := os1.Readlink(link)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got != target {
+		t.Errorf(`os1.Readlink("%s"): got %v, want %v`, link, got, target)
+	}
+}
+
+func TestShareNotExistError(t *testing.T) {
+	if testing.Short() {
+		t.Skip("slow test that uses network; skipping")
+	}
+	_, err := Stat(`\\no_such_server\no_such_share\no_such_file`)
+	if err == nil {
+		t.Fatal("Stat succeeded, but expected to fail")
+	}
+	if !os1.IsNotExist(err) {
+		t.Fatalf("os1.Stat failed with %q, but os1.IsNotExist(err) is false", err)
+	}
+}
+
+func TestStatPagefile(t *testing.T) {
+	_, err := Stat(`c:\pagefile.sys`)
+	if err == nil {
+		return
+	}
+	if os1.IsNotExist(err) {
+		t.Skip(`skipping because c:\pagefile.sys is not found`)
+	}
+	t.Fatal(err)
+}
+
+func TestStatSymlinkLoop(t *testing.T) {
+	err := os1.Symlink("x", "y")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os1.Remove("y")
+
+	err = os1.Symlink("y", "x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os1.Remove("x")
+
+	_, err = Stat("x")
+	if err != nil {
+		if _, ok := err.(*os1.PathError); !ok {
+			t.Errorf("expected *PathError, got %T: %v\n", err, err)
+		}
+	}
+}