From 25107c2e114be80740b7cd2af14aab3e2d9dfba2 Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 20 Feb 2018 12:21:12 -0800 Subject: [PATCH] Add NAS gateway support (#5516) --- .gitignore | 1 + cmd/bucket-policy.go | 6 +- cmd/fs-v1-metadata_test.go | 4 +- cmd/fs-v1-multipart.go | 26 +++---- cmd/fs-v1-multipart_test.go | 14 ++-- cmd/fs-v1.go | 90 +++++++++++----------- cmd/fs-v1_test.go | 20 ++--- cmd/gateway/gateway.go | 1 + cmd/gateway/nas/gateway-nas.go | 119 +++++++++++++++++++++++++++++ cmd/object-api-listobjects_test.go | 2 +- cmd/server-main.go | 2 +- cmd/server-main_test.go | 2 +- cmd/test-utils_test.go | 6 +- cmd/xl-sets.go | 4 +- cmd/xl-v1-bucket.go | 4 +- docs/gateway/azure.md | 2 +- docs/gateway/b2.md | 2 +- docs/gateway/gcs.md | 2 +- docs/gateway/manta.md | 2 +- docs/gateway/nas.md | 43 +++++++++++ docs/gateway/oss.md | 2 +- 21 files changed, 259 insertions(+), 95 deletions(-) create mode 100644 cmd/gateway/nas/gateway-nas.go create mode 100644 docs/gateway/nas.md diff --git a/.gitignore b/.gitignore index ca1800fd8..822ee70fe 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ prime/ snap/.snapcraft/ stage/ .sia_temp/ +buildcoveragecoverage.txt \ No newline at end of file diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go index a225f9338..eaddd5c28 100644 --- a/cmd/bucket-policy.go +++ b/cmd/bucket-policy.go @@ -90,7 +90,7 @@ func initBucketPolicies(objAPI ObjectLayer) (*bucketPolicies, error) { policies := make(map[string]policy.BucketAccessPolicy) // Loads bucket policy. for _, bucket := range buckets { - bp, pErr := readBucketPolicy(bucket.Name, objAPI) + bp, pErr := ReadBucketPolicy(bucket.Name, objAPI) if pErr != nil { // net.Dial fails for rpc client or any // other unexpected errors during net.Dial. @@ -130,9 +130,9 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader return &buffer, nil } -// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound +// ReadBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound // if bucket policy is not found. This function also parses the bucket policy into an object. -func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) { +func ReadBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) { // Read bucket policy JSON. bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI) if err != nil { diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index 27773cdf5..22fb9e441 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -44,7 +44,7 @@ func TestReadFSMetadata(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" @@ -79,7 +79,7 @@ func TestWriteFSMetadata(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 79802993e..b6af7efdd 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -36,22 +36,22 @@ import ( ) // Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID -func (fs *fsObjects) getUploadIDDir(bucket, object, uploadID string) string { +func (fs *FSObjects) getUploadIDDir(bucket, object, uploadID string) string { return pathJoin(fs.fsPath, minioMetaMultipartBucket, getSHA256Hash([]byte(pathJoin(bucket, object))), uploadID) } // Returns EXPORT/.minio.sys/multipart/SHA256 -func (fs *fsObjects) getMultipartSHADir(bucket, object string) string { +func (fs *FSObjects) getMultipartSHADir(bucket, object string) string { return pathJoin(fs.fsPath, minioMetaMultipartBucket, getSHA256Hash([]byte(pathJoin(bucket, object)))) } // Returns partNumber.etag -func (fs *fsObjects) encodePartFile(partNumber int, etag string) string { +func (fs *FSObjects) encodePartFile(partNumber int, etag string) string { return fmt.Sprintf("%.5d.%s", partNumber, etag) } // Returns partNumber and etag -func (fs *fsObjects) decodePartFile(name string) (partNumber int, etag string, err error) { +func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, err error) { result := strings.Split(name, ".") if len(result) != 2 { return 0, "", errUnexpected @@ -64,7 +64,7 @@ func (fs *fsObjects) decodePartFile(name string) (partNumber int, etag string, e } // Appends parts to an appendFile sequentially. -func (fs *fsObjects) backgroundAppend(bucket, object, uploadID string) { +func (fs *FSObjects) backgroundAppend(bucket, object, uploadID string) { fs.appendFileMapMu.Lock() file := fs.appendFileMap[uploadID] if file == nil { @@ -121,7 +121,7 @@ func (fs *fsObjects) backgroundAppend(bucket, object, uploadID string) { // ListMultipartUploads - lists all the uploadIDs for the specified object. // We do not support prefix based listing. -func (fs *fsObjects) ListMultipartUploads(bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, e error) { +func (fs *FSObjects) ListMultipartUploads(bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, e error) { if err := checkListMultipartArgs(bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil { return result, toObjectErr(errors.Trace(err)) } @@ -203,7 +203,7 @@ func (fs *fsObjects) ListMultipartUploads(bucket, object, keyMarker, uploadIDMar // subsequent request each UUID is unique. // // Implements S3 compatible initiate multipart API. -func (fs *fsObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) { +func (fs *FSObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) { if err := checkNewMultipartArgs(bucket, object, fs); err != nil { return "", toObjectErr(err, bucket) } @@ -238,7 +238,7 @@ func (fs *fsObjects) NewMultipartUpload(bucket, object string, meta map[string]s // CopyObjectPart - similar to PutObjectPart but reads data from an existing // object. Internally incoming data is written to '.minio.sys/tmp' location // and safely renamed to '.minio.sys/multipart' for reach parts. -func (fs *fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, +func (fs *FSObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string, srcEtag string) (pi PartInfo, e error) { if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil { @@ -277,7 +277,7 @@ func (fs *fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, // an ongoing multipart transaction. Internally incoming data is // written to '.minio.sys/tmp' location and safely renamed to // '.minio.sys/multipart' for reach parts. -func (fs *fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { +func (fs *FSObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { if err := checkPutObjectPartArgs(bucket, object, fs); err != nil { return pi, toObjectErr(errors.Trace(err), bucket) } @@ -358,7 +358,7 @@ func (fs *fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, // Implements S3 compatible ListObjectParts API. The resulting // ListPartsInfo structure is unmarshalled directly into XML and // replied back to the client. -func (fs *fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) { +func (fs *FSObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) { if err := checkListPartsArgs(bucket, object, fs); err != nil { return result, toObjectErr(errors.Trace(err)) } @@ -460,7 +460,7 @@ func (fs *fsObjects) ListObjectParts(bucket, object, uploadID string, partNumber // md5sums of all the parts. // // Implements S3 compatible Complete multipart API. -func (fs *fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) { +func (fs *FSObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) { if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil { return oi, toObjectErr(err) } @@ -634,7 +634,7 @@ func (fs *fsObjects) CompleteMultipartUpload(bucket string, object string, uploa // that this is an atomic idempotent operation. Subsequent calls have // no affect and further requests to the same uploadID would not be // honored. -func (fs *fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error { +func (fs *FSObjects) AbortMultipartUpload(bucket, object, uploadID string) error { if err := checkAbortMultipartArgs(bucket, object, fs); err != nil { return err } @@ -666,7 +666,7 @@ func (fs *fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error // Removes multipart uploads if any older than `expiry` duration // on all buckets for every `cleanupInterval`, this function is // blocking and should be run in a go-routine. -func (fs *fsObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) { +func (fs *FSObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) { ticker := time.NewTicker(cleanupInterval) for { select { diff --git a/cmd/fs-v1-multipart_test.go b/cmd/fs-v1-multipart_test.go index d56b5331b..04544b21f 100644 --- a/cmd/fs-v1-multipart_test.go +++ b/cmd/fs-v1-multipart_test.go @@ -33,7 +33,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) // Close the go-routine, we are going to // manually start it and test in this test case. @@ -73,7 +73,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" @@ -102,7 +102,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) { disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" data := []byte("12345") @@ -134,7 +134,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" data := []byte("12345") @@ -166,7 +166,7 @@ func TestCompleteMultipartUpload(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" data := []byte("12345") @@ -200,7 +200,7 @@ func TestAbortMultipartUpload(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" data := []byte("12345") @@ -233,7 +233,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) { obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index f918e33f2..a2228bcd9 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -37,8 +37,8 @@ import ( "github.com/minio/minio/pkg/madmin" ) -// fsObjects - Implements fs object layer. -type fsObjects struct { +// FSObjects - Implements fs object layer. +type FSObjects struct { // Path to be exported over S3 API. fsPath string @@ -93,8 +93,8 @@ func initMetaVolumeFS(fsPath, fsUUID string) error { } -// newFSObjectLayer - initialize new fs object layer. -func newFSObjectLayer(fsPath string) (ObjectLayer, error) { +// NewFSObjectLayer - initialize new fs object layer. +func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { if fsPath == "" { return nil, errInvalidArgument } @@ -146,7 +146,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { } // Initialize fs objects. - fs := &fsObjects{ + fs := &FSObjects{ fsPath: fsPath, fsUUID: fsUUID, rwPool: &fsIOPool{ @@ -180,8 +180,8 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { return fs, nil } -// Should be called when process shuts down. -func (fs *fsObjects) Shutdown() error { +// Shutdown - should be called when process shuts down. +func (fs *FSObjects) Shutdown() error { fs.fsFormatRlk.Close() // Cleanup and delete tmp uuid. @@ -189,7 +189,7 @@ func (fs *fsObjects) Shutdown() error { } // StorageInfo - returns underlying storage statistics. -func (fs *fsObjects) StorageInfo() StorageInfo { +func (fs *FSObjects) StorageInfo() StorageInfo { info, err := getDiskInfo((fs.fsPath)) errorIf(err, "Unable to get disk info %#v", fs.fsPath) storageInfo := StorageInfo{ @@ -202,13 +202,13 @@ func (fs *fsObjects) StorageInfo() StorageInfo { // Locking operations -// List namespace locks held in object layer -func (fs *fsObjects) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { +// ListLocks - List namespace locks held in object layer +func (fs *FSObjects) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { return []VolumeLockInfo{}, NotImplemented{} } -// Clear namespace locks held in object layer -func (fs *fsObjects) ClearLocks([]VolumeLockInfo) error { +// ClearLocks - Clear namespace locks held in object layer +func (fs *FSObjects) ClearLocks([]VolumeLockInfo) error { return NotImplemented{} } @@ -217,7 +217,7 @@ func (fs *fsObjects) ClearLocks([]VolumeLockInfo) error { // getBucketDir - will convert incoming bucket names to // corresponding valid bucket names on the backend in a platform // compatible way for all operating systems. -func (fs *fsObjects) getBucketDir(bucket string) (string, error) { +func (fs *FSObjects) getBucketDir(bucket string) (string, error) { // Verify if bucket is valid. if !IsValidBucketName(bucket) { return "", errors.Trace(BucketNameInvalid{Bucket: bucket}) @@ -227,7 +227,7 @@ func (fs *fsObjects) getBucketDir(bucket string) (string, error) { return bucketDir, nil } -func (fs *fsObjects) statBucketDir(bucket string) (os.FileInfo, error) { +func (fs *FSObjects) statBucketDir(bucket string) (os.FileInfo, error) { bucketDir, err := fs.getBucketDir(bucket) if err != nil { return nil, err @@ -239,9 +239,9 @@ func (fs *fsObjects) statBucketDir(bucket string) (os.FileInfo, error) { return st, nil } -// MakeBucket - create a new bucket, returns if it +// MakeBucketWithLocation - create a new bucket, returns if it // already exists. -func (fs *fsObjects) MakeBucketWithLocation(bucket, location string) error { +func (fs *FSObjects) MakeBucketWithLocation(bucket, location string) error { bucketLock := fs.nsMutex.NewNSLock(bucket, "") if err := bucketLock.GetLock(globalObjectTimeout); err != nil { return err @@ -260,7 +260,7 @@ func (fs *fsObjects) MakeBucketWithLocation(bucket, location string) error { } // GetBucketInfo - fetch bucket metadata info. -func (fs *fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) { +func (fs *FSObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) { bucketLock := fs.nsMutex.NewNSLock(bucket, "") if e := bucketLock.GetRLock(globalObjectTimeout); e != nil { return bi, e @@ -280,7 +280,7 @@ func (fs *fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) { } // ListBuckets - list all s3 compatible buckets (directories) at fsPath. -func (fs *fsObjects) ListBuckets() ([]BucketInfo, error) { +func (fs *FSObjects) ListBuckets() ([]BucketInfo, error) { if err := checkPathLength(fs.fsPath); err != nil { return nil, errors.Trace(err) } @@ -321,7 +321,7 @@ func (fs *fsObjects) ListBuckets() ([]BucketInfo, error) { // DeleteBucket - delete a bucket and all the metadata associated // with the bucket including pending multipart, object metadata. -func (fs *fsObjects) DeleteBucket(bucket string) error { +func (fs *FSObjects) DeleteBucket(bucket string) error { bucketLock := fs.nsMutex.NewNSLock(bucket, "") if err := bucketLock.GetLock(globalObjectTimeout); err != nil { return err @@ -360,7 +360,7 @@ func (fs *fsObjects) DeleteBucket(bucket string) error { // CopyObject - copy object source object to destination object. // if source object and destination object are same we only // update metadata. -func (fs *fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string, srcEtag string) (oi ObjectInfo, e error) { +func (fs *FSObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string, srcEtag string) (oi ObjectInfo, e error) { cpSrcDstSame := srcBucket == dstBucket && srcObject == dstObject // Hold write lock on destination since in both cases // - if source and destination are same @@ -463,7 +463,7 @@ func (fs *fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject strin // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (fs *fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { +func (fs *FSObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { if err = checkGetObjArgs(bucket, object); err != nil { return err } @@ -478,7 +478,7 @@ func (fs *fsObjects) GetObject(bucket, object string, offset int64, length int64 } // getObject - wrapper for GetObject -func (fs *fsObjects) getObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { +func (fs *FSObjects) getObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { if _, err = fs.statBucketDir(bucket); err != nil { return toObjectErr(err, bucket) } @@ -549,7 +549,7 @@ func (fs *fsObjects) getObject(bucket, object string, offset int64, length int64 } // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. -func (fs *fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) { +func (fs *FSObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) { fsMeta := fsMetaV1{} fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object)) if err != nil && errors.Cause(err) != errFileAccessDenied { @@ -597,7 +597,7 @@ func (fs *fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e erro } // GetObjectInfo - reads object metadata and replies back ObjectInfo. -func (fs *fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) { +func (fs *FSObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) { // Lock the object before reading. objectLock := fs.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetRLock(globalObjectTimeout); err != nil { @@ -619,7 +619,7 @@ func (fs *fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e erro // This function does the following check, suppose // object is "a/b/c/d", stat makes sure that objects ""a/b/c"" // "a/b" and "a" do not exist. -func (fs *fsObjects) parentDirIsObject(bucket, parent string) bool { +func (fs *FSObjects) parentDirIsObject(bucket, parent string) bool { var isParentDirObject func(string) bool isParentDirObject = func(p string) bool { if p == "." || p == "/" { @@ -640,7 +640,7 @@ func (fs *fsObjects) parentDirIsObject(bucket, parent string) bool { // until EOF, writes data directly to configured filesystem path. // Additionally writes `fs.json` which carries the necessary metadata // for future object operations. -func (fs *fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { +func (fs *FSObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { if err := checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil { return ObjectInfo{}, err } @@ -654,7 +654,7 @@ func (fs *fsObjects) PutObject(bucket string, object string, data *hash.Reader, } // putObject - wrapper for PutObject -func (fs *fsObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { +func (fs *FSObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { // No metadata is set, allocate a new one. if metadata == nil { metadata = make(map[string]string) @@ -778,7 +778,7 @@ func (fs *fsObjects) putObject(bucket string, object string, data *hash.Reader, // DeleteObject - deletes an object from a bucket, this operation is destructive // and there are no rollbacks supported. -func (fs *fsObjects) DeleteObject(bucket, object string) error { +func (fs *FSObjects) DeleteObject(bucket, object string) error { // Acquire a write lock before deleting the object. objectLock := fs.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetLock(globalOperationTimeout); err != nil { @@ -825,7 +825,7 @@ func (fs *fsObjects) DeleteObject(bucket, object string) error { // Returns function "listDir" of the type listDirFunc. // isLeaf - is used by listDir function to check if an entry // is a leaf or non-leaf entry. -func (fs *fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { +func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { // listDir - lists all the entries at a given prefix and given entry in the prefix. listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error) { entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir)) @@ -842,7 +842,7 @@ func (fs *fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { // getObjectETag is a helper function, which returns only the md5sum // of the file on the disk. -func (fs *fsObjects) getObjectETag(bucket, entry string) (string, error) { +func (fs *FSObjects) getObjectETag(bucket, entry string) (string, error) { fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile) // Read `fs.json` to perhaps contend with @@ -891,7 +891,7 @@ func (fs *fsObjects) getObjectETag(bucket, entry string) (string, error) { // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool // state for future re-entrant list requests. -func (fs *fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { +func (fs *FSObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { if err := checkListObjsArgs(bucket, prefix, marker, delimiter, fs); err != nil { return loi, err } @@ -1051,53 +1051,53 @@ func (fs *fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKe } // HealFormat - no-op for fs, Valid only for XL. -func (fs *fsObjects) HealFormat(dryRun bool) (madmin.HealResultItem, error) { +func (fs *FSObjects) HealFormat(dryRun bool) (madmin.HealResultItem, error) { return madmin.HealResultItem{}, errors.Trace(NotImplemented{}) } // HealObject - no-op for fs. Valid only for XL. -func (fs *fsObjects) HealObject(bucket, object string, dryRun bool) ( +func (fs *FSObjects) HealObject(bucket, object string, dryRun bool) ( res madmin.HealResultItem, err error) { return res, errors.Trace(NotImplemented{}) } // HealBucket - no-op for fs, Valid only for XL. -func (fs *fsObjects) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, +func (fs *FSObjects) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, error) { return nil, errors.Trace(NotImplemented{}) } // ListObjectsHeal - list all objects to be healed. Valid only for XL -func (fs *fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { +func (fs *FSObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { return loi, errors.Trace(NotImplemented{}) } // ListBucketsHeal - list all buckets to be healed. Valid only for XL -func (fs *fsObjects) ListBucketsHeal() ([]BucketInfo, error) { +func (fs *FSObjects) ListBucketsHeal() ([]BucketInfo, error) { return []BucketInfo{}, errors.Trace(NotImplemented{}) } // SetBucketPolicy sets policy on bucket -func (fs *fsObjects) SetBucketPolicy(bucket string, policy policy.BucketAccessPolicy) error { +func (fs *FSObjects) SetBucketPolicy(bucket string, policy policy.BucketAccessPolicy) error { return persistAndNotifyBucketPolicyChange(bucket, false, policy, fs) } // GetBucketPolicy will get policy on bucket -func (fs *fsObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (fs *FSObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { policy := fs.bucketPolicies.GetBucketPolicy(bucket) if reflect.DeepEqual(policy, emptyBucketPolicy) { - return readBucketPolicy(bucket, fs) + return ReadBucketPolicy(bucket, fs) } return policy, nil } // DeleteBucketPolicy deletes all policies on bucket -func (fs *fsObjects) DeleteBucketPolicy(bucket string) error { +func (fs *FSObjects) DeleteBucketPolicy(bucket string) error { return persistAndNotifyBucketPolicyChange(bucket, true, emptyBucketPolicy, fs) } // ListObjectsV2 lists all blobs in bucket filtered by prefix -func (fs *fsObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { +func (fs *FSObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { loi, err := fs.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys) if err != nil { return result, err @@ -1114,8 +1114,8 @@ func (fs *fsObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter } // RefreshBucketPolicy refreshes cache policy with what's on disk. -func (fs *fsObjects) RefreshBucketPolicy(bucket string) error { - policy, err := readBucketPolicy(bucket, fs) +func (fs *FSObjects) RefreshBucketPolicy(bucket string) error { + policy, err := ReadBucketPolicy(bucket, fs) if err != nil { if reflect.DeepEqual(policy, emptyBucketPolicy) { @@ -1127,11 +1127,11 @@ func (fs *fsObjects) RefreshBucketPolicy(bucket string) error { } // IsNotificationSupported returns whether bucket notification is applicable for this layer. -func (fs *fsObjects) IsNotificationSupported() bool { +func (fs *FSObjects) IsNotificationSupported() bool { return true } // IsEncryptionSupported returns whether server side encryption is applicable for this layer. -func (fs *fsObjects) IsEncryptionSupported() bool { +func (fs *FSObjects) IsEncryptionSupported() bool { return true } diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 2f6c7d47d..e8f849341 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -56,7 +56,7 @@ func TestFSParentDirIsObject(t *testing.T) { t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName) } - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) testCases := []struct { parentIsObject bool objectName string @@ -101,16 +101,16 @@ func TestFSParentDirIsObject(t *testing.T) { // and constructs a valid `FS` object layer. func TestNewFS(t *testing.T) { // Do not attempt to create this path, the test validates - // so that newFSObjectLayer initializes non existing paths + // so that NewFSObjectLayer initializes non existing paths // and successfully returns initialized object layer. disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) defer os.RemoveAll(disk) - _, err := newFSObjectLayer("") + _, err := NewFSObjectLayer("") if err != errInvalidArgument { t.Errorf("Expecting error invalid argument, got %s", err) } - _, err = newFSObjectLayer(disk) + _, err = NewFSObjectLayer(disk) if err != nil { errMsg := "Unable to recognize backend format, Disk is not in FS format." if err.Error() == errMsg { @@ -131,10 +131,10 @@ func TestFSShutdown(t *testing.T) { bucketName := "testbucket" objectName := "object" // Create and return an fsObject with its path in the disk - prepareTest := func() (*fsObjects, string) { + prepareTest := func() (*FSObjects, string) { disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) objectContent := "12345" obj.MakeBucketWithLocation(bucketName, "") obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil) @@ -164,7 +164,7 @@ func TestFSGetBucketInfo(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" obj.MakeBucketWithLocation(bucketName, "") @@ -266,7 +266,7 @@ func TestFSDeleteObject(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" objectName := "object" @@ -311,7 +311,7 @@ func TestFSDeleteBucket(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" err := obj.MakeBucketWithLocation(bucketName, "") @@ -350,7 +350,7 @@ func TestFSListBuckets(t *testing.T) { defer os.RemoveAll(disk) obj := initFSObjects(disk, t) - fs := obj.(*fsObjects) + fs := obj.(*FSObjects) bucketName := "bucket" if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil { diff --git a/cmd/gateway/gateway.go b/cmd/gateway/gateway.go index 4454a5e52..656f56c4b 100644 --- a/cmd/gateway/gateway.go +++ b/cmd/gateway/gateway.go @@ -22,6 +22,7 @@ import ( _ "github.com/minio/minio/cmd/gateway/b2" _ "github.com/minio/minio/cmd/gateway/gcs" _ "github.com/minio/minio/cmd/gateway/manta" + _ "github.com/minio/minio/cmd/gateway/nas" _ "github.com/minio/minio/cmd/gateway/oss" _ "github.com/minio/minio/cmd/gateway/s3" _ "github.com/minio/minio/cmd/gateway/sia" diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go new file mode 100644 index 000000000..7c16796c1 --- /dev/null +++ b/cmd/gateway/nas/gateway-nas.go @@ -0,0 +1,119 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nas + +import ( + "github.com/minio/cli" + "github.com/minio/minio-go/pkg/policy" + minio "github.com/minio/minio/cmd" + "github.com/minio/minio/pkg/auth" +) + +const ( + nasBackend = "nas" +) + +func init() { + const nasGatewayTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} PATH +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +PATH: + path to NAS mount point. + +ENVIRONMENT VARIABLES: + ACCESS: + MINIO_ACCESS_KEY: Username or access key of minimum 3 characters in length. + MINIO_SECRET_KEY: Password or secret key of minimum 8 characters in length. + + BROWSER: + MINIO_BROWSER: To disable web browser access, set this value to "off". + + UPDATE: + MINIO_UPDATE: To turn off in-place upgrades, set this value to "off". + +EXAMPLES: + 1. Start minio gateway server for NAS backend. + $ export MINIO_ACCESS_KEY=accesskey + $ export MINIO_SECRET_KEY=secretkey + $ {{.HelpName}} /shared/nasvol +` + + minio.RegisterGatewayCommand(cli.Command{ + Name: nasBackend, + Usage: "Network-attached storage (NAS).", + Action: nasGatewayMain, + CustomHelpTemplate: nasGatewayTemplate, + HideHelpCommand: true, + }) +} + +// Handler for 'minio gateway nas' command line. +func nasGatewayMain(ctx *cli.Context) { + // Validate gateway arguments. + host := ctx.Args().First() + if host == "" { + cli.ShowCommandHelpAndExit(ctx, "nas", 1) + } + // Validate gateway arguments. + minio.StartGateway(ctx, &NAS{host}) +} + +// NAS implements Gateway. +type NAS struct { + host string +} + +// Name implements Gateway interface. +func (g *NAS) Name() string { + return nasBackend +} + +// NewGatewayLayer returns nas gatewaylayer. +func (g *NAS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) { + var err error + newObject, err := minio.NewFSObjectLayer(g.host) + if err != nil { + return nil, err + } + return &nasObjects{newObject.(*minio.FSObjects)}, nil +} + +// Production - nas gateway is production ready. +func (g *NAS) Production() bool { + return true +} + +// nasObjects implements gateway for Minio and S3 compatible object storage servers. +type nasObjects struct { + *minio.FSObjects +} + +// IsNotificationSupported returns whether notifications are applicable for this layer. +func (l *nasObjects) IsNotificationSupported() bool { + return false +} + +// GetBucketPolicy will get policy on bucket +func (l *nasObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { + return minio.ReadBucketPolicy(bucket, l) +} diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 2344c26e2..409f13287 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -576,7 +576,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { // Initialize FS backend for the benchmark. func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) { var err error - obj, err = newFSObjectLayer(disk) + obj, err = NewFSObjectLayer(disk) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/server-main.go b/cmd/server-main.go index de3e33215..091f153c1 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -270,7 +270,7 @@ func newObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err error) { isFS := len(endpoints) == 1 if isFS { // Initialize new FS object layer. - return newFSObjectLayer(endpoints[0].Path) + return NewFSObjectLayer(endpoints[0].Path) } format, err := waitForFormatXL(endpoints[0].IsLocal, endpoints, globalXLSetCount, globalXLSetDriveCount) diff --git a/cmd/server-main_test.go b/cmd/server-main_test.go index 1334ba152..663090407 100644 --- a/cmd/server-main_test.go +++ b/cmd/server-main_test.go @@ -36,7 +36,7 @@ func TestNewObjectLayer(t *testing.T) { if err != nil { t.Fatal("Unexpected object layer initialization error", err) } - _, ok := obj.(*fsObjects) + _, ok := obj.(*FSObjects) if !ok { t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj)) } diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index d624dfba6..b9c27d4db 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -163,7 +163,7 @@ func prepareFS() (ObjectLayer, string, error) { if err != nil { return nil, "", err } - obj, err := newFSObjectLayer(fsDirs[0]) + obj, err := NewFSObjectLayer(fsDirs[0]) if err != nil { return nil, "", err } @@ -221,7 +221,7 @@ func prepareXL16() (ObjectLayer, []string, error) { func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) { newTestConfig(globalMinioDefaultRegion) var err error - obj, err = newFSObjectLayer(disk) + obj, err = NewFSObjectLayer(disk) if err != nil { t.Fatal(err) } @@ -1685,7 +1685,7 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro isFS := len(endpoints) == 1 if isFS { // Initialize new FS object layer. - return newFSObjectLayer(endpoints[0].Path) + return NewFSObjectLayer(endpoints[0].Path) } _, err = waitForFormatXL(endpoints[0].IsLocal, endpoints, 1, 16) diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index fb784f448..8144df8e8 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -433,7 +433,7 @@ func (s *xlSets) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, erro // fetch bucket policy from cache. bpolicy := s.bucketPolicies.GetBucketPolicy(bucket) if reflect.DeepEqual(bpolicy, emptyBucketPolicy) { - return readBucketPolicy(bucket, s) + return ReadBucketPolicy(bucket, s) } return bpolicy, nil } @@ -445,7 +445,7 @@ func (s *xlSets) DeleteBucketPolicy(bucket string) error { // RefreshBucketPolicy refreshes policy cache from disk func (s *xlSets) RefreshBucketPolicy(bucket string) error { - policy, err := readBucketPolicy(bucket, s) + policy, err := ReadBucketPolicy(bucket, s) if err != nil { if reflect.DeepEqual(policy, emptyBucketPolicy) { return s.bucketPolicies.DeleteBucketPolicy(bucket) diff --git a/cmd/xl-v1-bucket.go b/cmd/xl-v1-bucket.go index 54197d8e1..f36e295ae 100644 --- a/cmd/xl-v1-bucket.go +++ b/cmd/xl-v1-bucket.go @@ -291,7 +291,7 @@ func (xl xlObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, e // fetch bucket policy from cache. bpolicy := xl.bucketPolicies.GetBucketPolicy(bucket) if reflect.DeepEqual(bpolicy, emptyBucketPolicy) { - return readBucketPolicy(bucket, xl) + return ReadBucketPolicy(bucket, xl) } return bpolicy, nil } @@ -303,7 +303,7 @@ func (xl xlObjects) DeleteBucketPolicy(bucket string) error { // RefreshBucketPolicy refreshes policy cache from disk func (xl xlObjects) RefreshBucketPolicy(bucket string) error { - policy, err := readBucketPolicy(bucket, xl) + policy, err := ReadBucketPolicy(bucket, xl) if err != nil { if reflect.DeepEqual(policy, emptyBucketPolicy) { diff --git a/docs/gateway/azure.md b/docs/gateway/azure.md index 3ef710ca0..9e915a477 100644 --- a/docs/gateway/azure.md +++ b/docs/gateway/azure.md @@ -17,7 +17,7 @@ export MINIO_SECRET_KEY=azureaccountkey minio gateway azure ``` ## Test using Minio Browser -Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully. +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser-gateway.png?raw=true) ## Test using Minio Client `mc` diff --git a/docs/gateway/b2.md b/docs/gateway/b2.md index f413be4d2..b0ed41cb1 100644 --- a/docs/gateway/b2.md +++ b/docs/gateway/b2.md @@ -13,7 +13,7 @@ docker run -p 9000:9000 --name b2-s3 \ ``` ## Test using Minio Browser -Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully. +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. ![Screenshot](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/minio-browser-gateway.png) diff --git a/docs/gateway/gcs.md b/docs/gateway/gcs.md index cbb825b21..20bc331df 100644 --- a/docs/gateway/gcs.md +++ b/docs/gateway/gcs.md @@ -32,7 +32,7 @@ minio gateway gcs yourprojectid ``` ## Test using Minio Browser -Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully. +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser-gateway.png?raw=true) diff --git a/docs/gateway/manta.md b/docs/gateway/manta.md index 70d745613..e3073f0a0 100644 --- a/docs/gateway/manta.md +++ b/docs/gateway/manta.md @@ -21,7 +21,7 @@ export MANTA_SUBUSER=devuser minio gateway manta ``` ## Test using Minio Browser -Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully. +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser-gateway.png?raw=true) ## Test using Minio Client `mc` diff --git a/docs/gateway/nas.md b/docs/gateway/nas.md new file mode 100644 index 000000000..724def924 --- /dev/null +++ b/docs/gateway/nas.md @@ -0,0 +1,43 @@ +# Minio NAS Gateway [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) +Minio Gateway adds Amazon S3 compatibility to NAS storage. You may run multiple minio instances on the same shared NAS volume as a distributed object gateway. + +## Run Minio Gateway for NAS Storage +### Using Docker +``` +docker run -p 9000:9000 --name nas-s3 \ + -e "MINIO_ACCESS_KEY=minio" \ + -e "MINIO_SECRET_KEY=minio123" \ + minio/minio:edge gateway nas /shared/nasvol +``` + +### Using Binary +``` +export MINIO_ACCESS_KEY=minioaccesskey +export MINIO_SECRET_KEY=miniosecretkey +minio gateway nas /shared/nasvol +``` +## Test using Minio Browser +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. + +![Screenshot](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/minio-browser-gateway.png) + +## Test using Minio Client `mc` +`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. + +### Configure `mc` +``` +mc config host add mynas http://gateway-ip:9000 access_key secret_key +``` + +### List buckets on nas +``` +mc ls mynas +[2017-02-22 01:50:43 PST] 0B ferenginar/ +[2017-02-26 21:43:51 PST] 0B my-bucket/ +[2017-02-26 22:10:11 PST] 0B test-bucket1/ +``` + +## Explore Further +- [`mc` command-line interface](https://docs.minio.io/docs/minio-client-quickstart-guide) +- [`aws` command-line interface](https://docs.minio.io/docs/aws-cli-with-minio) +- [`minio-go` Go SDK](https://docs.minio.io/docs/golang-client-quickstart-guide) diff --git a/docs/gateway/oss.md b/docs/gateway/oss.md index 77dbcc998..015e143c1 100644 --- a/docs/gateway/oss.md +++ b/docs/gateway/oss.md @@ -19,7 +19,7 @@ minio gateway azure ``` ## Test using Minio Browser -Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully. +Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. ![Screenshot](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/minio-browser-gateway.png)