diff --git a/README.md b/README.md index 4cb0fdb17..89c7215cc 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Object API (Amazon S3 compatible): Open a web browser and navigate to http://127.0.0.1:9000 to view your buckets on minio server. -![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/Minio_Browser.jpg?raw=true) +![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true) ## 4. Test Minio Server using `mc` diff --git a/docs/backend-multipart.md b/docs/backend-multipart.md deleted file mode 100644 index 465c33f69..000000000 --- a/docs/backend-multipart.md +++ /dev/null @@ -1,33 +0,0 @@ -## Multipart - -Each incoming part is uploaded in the following format. - -Placeholder uploads.json to indicate a leaf. - - ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/uploads.json``` - -Incomplete file - - ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/UPLOADID/00000.incomplete``` - -Actual parts - - ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/UPLOADID/PART_NUMBER.MD5SUM_STRING``` - -## FS Format. - -Each of these parts are concatenated back to a single contigous file. - -```EXPORT_DIR/BUCKET/PATH/TO/OBJECT``` - -## XL Format. - -Each of these parts are kept as is in the following format. - -Special json file indicate the metata multipart information, essentially list of parts etc. - -```EXPORT_DIR/BUCKET/PATH/TO/OBJECT/00000.minio.multipart``` - -All the parts that were uploaded. - -```EXPORT_DIR/BUCKET/PATH/TO/OBJECT/PART_NUMBER.minio.multipart``` diff --git a/docs/backend/README.md b/docs/backend/README.md index e69de29bb..c0412b288 100644 --- a/docs/backend/README.md +++ b/docs/backend/README.md @@ -0,0 +1,6 @@ +## Backends + +Minio currently implements two types of backends namely. + +- Filesystem layer (fs). +- ErasureCode layer (XL). diff --git a/docs/backend/fs/README.md b/docs/backend/fs/README.md new file mode 100644 index 000000000..1c16d02bf --- /dev/null +++ b/docs/backend/fs/README.md @@ -0,0 +1,25 @@ +### Backend format `fs.json` + +```go +// objectPartInfo Info of each part kept in the multipart metadata +// file after CompleteMultipartUpload() is called. +type objectPartInfo struct { + Number int `json:"number"` + Name string `json:"name"` + ETag string `json:"etag"` + Size int64 `json:"size"` +} + +// A fsMetaV1 represents a metadata header mapping keys to sets of values. +type fsMetaV1 struct { + Version string `json:"version"` + Format string `json:"format"` + Minio struct { + Release string `json:"release"` + } `json:"minio"` + // Metadata map for current object `fs.json`. + Meta map[string]string `json:"meta,omitempty"` + Parts []objectPartInfo `json:"parts,omitempty"` +} + +``` diff --git a/docs/backend/fs/fs.json b/docs/backend/fs/fs.json index c2a7c0c19..6b309dde3 100644 --- a/docs/backend/fs/fs.json +++ b/docs/backend/fs/fs.json @@ -4,6 +4,10 @@ "minio": { "release": "DEVELOPMENT.GOGET" }, + "meta": { + "content-type": "binary/octet-stream", + "content-encoding": "gzip" + }, "parts": [ { "number": 1, diff --git a/docs/backend/xl/README.md b/docs/backend/xl/README.md new file mode 100644 index 000000000..27920ef5c --- /dev/null +++ b/docs/backend/xl/README.md @@ -0,0 +1,54 @@ +### Backend format `xl.json` + +```go +// objectPartInfo Info of each part kept in the multipart metadata +// file after CompleteMultipartUpload() is called. +type objectPartInfo struct { + Number int `json:"number"` + Name string `json:"name"` + ETag string `json:"etag"` + Size int64 `json:"size"` +} + +// checkSumInfo - carries checksums of individual scattered parts per disk. +type checkSumInfo struct { + Name string `json:"name"` + Algorithm string `json:"algorithm"` + Hash string `json:"hash"` +} + +// erasureInfo - carries erasure coding related information, block +// distribution and checksums. +type erasureInfo struct { + Algorithm string `json:"algorithm"` + DataBlocks int `json:"data"` + ParityBlocks int `json:"parity"` + BlockSize int64 `json:"blockSize"` + Index int `json:"index"` + Distribution []int `json:"distribution"` + Checksum []checkSumInfo `json:"checksum,omitempty"` +} + +// statInfo - carries stat information of the object. +type statInfo struct { + Size int64 `json:"size"` // Size of the object `xl.json`. + ModTime time.Time `json:"modTime"` // ModTime of the object `xl.json`. +} + +// A xlMetaV1 represents `xl.json` metadata header. +type xlMetaV1 struct { + Version string `json:"version"` // Version of the current `xl.json`. + Format string `json:"format"` // Format of the current `xl.json`. + Stat statInfo `json:"stat"` // Stat of the current object `xl.json`. + // Erasure coded info for the current object `xl.json`. + Erasure erasureInfo `json:"erasure"` + // Minio release tag for current object `xl.json`. + Minio struct { + Release string `json:"release"` + } `json:"minio"` + // Metadata map for current object `xl.json`. + Meta map[string]string `json:"meta,omitempty"` + // Captures all the individual object `xl.json`. + Parts []objectPartInfo `json:"parts,omitempty"` +} +``` diff --git a/docs/backend/xl/xl.json b/docs/backend/xl/xl.json index eda47e2cf..9cdc3ca06 100644 --- a/docs/backend/xl/xl.json +++ b/docs/backend/xl/xl.json @@ -16,7 +16,7 @@ "meta": { "md5Sum": "97586a5290d4f5a41328062d6a7da593-3", "content-type": "application\/octet-stream", - "content-encoding": "" + "content-encoding": "gzip" }, "minio": { "release": "DEVELOPMENT.GOGET" diff --git a/docs/web-browser.md b/docs/browser/README.md similarity index 100% rename from docs/web-browser.md rename to docs/browser/README.md diff --git a/docs/api-bucket-policy.md b/docs/bucket/policy/README.md similarity index 100% rename from docs/api-bucket-policy.md rename to docs/bucket/policy/README.md diff --git a/docs/caching.md b/docs/caching/README.md similarity index 100% rename from docs/caching.md rename to docs/caching/README.md diff --git a/docs/Minio-erasure-code-quickStart-guide.md b/docs/erasure/README.md similarity index 98% rename from docs/Minio-erasure-code-quickStart-guide.md rename to docs/erasure/README.md index b6ad85346..1d7611f73 100644 --- a/docs/Minio-erasure-code-quickStart-guide.md +++ b/docs/erasure/README.md @@ -14,7 +14,7 @@ Erasure code is a mathematical algorithm to reconstruct missing or corrupted dat Erasure code protects data from multiple drives failure unlike RAID or replication. Minio encodes each object individually with a high parity count. Storage servers once deployed should not require drive replacement or healing for the lifetime of the server. Minio's erasure coded backend is designed for operational efficiency and takes full advantage of hardware acceleration whenever available. -[![Erasure](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/Erasure_Code.jpg?raw=true)] +[![Erasure](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/erasure-code.jpg?raw=true)] ## What is Bit Rot protection? diff --git a/docs/logging.md b/docs/logging.md deleted file mode 100644 index 8554ccfcc..000000000 --- a/docs/logging.md +++ /dev/null @@ -1,28 +0,0 @@ -### Logging. - -- `fatalIf` - wrapper function which takes error and prints jsonic error messages. -- `errorIf` - similar to fatalIf but doesn't exit on err != nil. - -Supported logging targets. - -- console -- file -- syslog - -Sample logger section from `~/.minio/config.json` -``` - "console": { - "enable": true, - "level": "error" - }, - "file": { - "enable": false, - "fileName": "", - "level": "error" - }, - "syslog": { - "enable": false, - "address": "", - "level": "error" - } -``` diff --git a/docs/screenshots/Erasure_Code.jpg b/docs/screenshots/erasure-code.jpg similarity index 100% rename from docs/screenshots/Erasure_Code.jpg rename to docs/screenshots/erasure-code.jpg diff --git a/docs/screenshots/Minio_Browser.jpg b/docs/screenshots/minio-browser.jpg similarity index 100% rename from docs/screenshots/Minio_Browser.jpg rename to docs/screenshots/minio-browser.jpg diff --git a/docs/xl-file-json.md b/docs/xl-file-json.md deleted file mode 100644 index 4bb55273a..000000000 --- a/docs/xl-file-json.md +++ /dev/null @@ -1,41 +0,0 @@ -### file.json - -``file.json`` is a special file captured and written by XL storage API layer -to interpret, manage and extract erasured data from multiple disks. - -```json -{ - "version": "1.0.0", - "stat": { - "size": 24256, - "modTime": "2016-04-28T00:11:37.843Z", - "delete": false, - "version": 0 - }, - "erasure": { - "data": 5, - "parity": 5, - "blockSize": 4194304 - ], - "minio": { - "release": "RELEASE.2016-04-28T00-09-47Z" - } -} -``` - -#### JSON meaning. - -- "version" // Version of the meta json file. - -- "stat" // Stat value of written file. - - - "size" // Size of the file. - - "modTime" // Modified time of the file. - - "deleted" // Field to track if the file is deleted when disks are down. - - "version" // File version tracked when disks are down. - -- "erasure" // Erasure metadata for the written file. - - - "data" // Data blocks parts of the file. - - "parity" // Parity blocks parts of the file. - - "blockSize" // BlockSize read/write chunk size. diff --git a/queues.go b/queues.go index cbcf1b269..ba63b2875 100644 --- a/queues.go +++ b/queues.go @@ -112,7 +112,7 @@ func notifyObjectCreatedEvent(nConfig notificationConfig, eventType EventName, b // Following blocks fills in all the necessary details of s3 event message structure. // http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html events := []*NotificationEvent{ - &NotificationEvent{ + { EventVersion: "2.0", EventSource: "aws:s3", AwsRegion: region, @@ -158,7 +158,7 @@ func notifyObjectDeletedEvent(nConfig notificationConfig, bucket string, object // Following blocks fills in all the necessary details of s3 event message structure. // http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html events := []*NotificationEvent{ - &NotificationEvent{ + { EventVersion: "2.0", EventSource: "aws:s3", AwsRegion: region, diff --git a/xl-v1-metadata.go b/xl-v1-metadata.go index f92b91917..d5e013975 100644 --- a/xl-v1-metadata.go +++ b/xl-v1-metadata.go @@ -27,7 +27,6 @@ import ( const ( // Erasure related constants. erasureAlgorithmKlauspost = "klauspost/reedsolomon/vandermonde" - erasureAlgorithmISAL = "isa-l/reedsolomon/cauchy" ) // objectPartInfo Info of each part kept in the multipart metadata