From 721ec9ef5aa9eadb72bc6ff013a6782beaa11bc2 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 3 Mar 2015 00:54:57 -0800 Subject: [PATCH 1/3] Force exported functions to have comments --- buildscripts/deadcode.go | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/buildscripts/deadcode.go b/buildscripts/deadcode.go index bcb40e211..76327a33e 100644 --- a/buildscripts/deadcode.go +++ b/buildscripts/deadcode.go @@ -73,7 +73,7 @@ func doDir(name string) { return false } fs := token.NewFileSet() - pkgs, err := parser.ParseDir(fs, name, notests, parser.Mode(0)) + pkgs, err := parser.ParseDir(fs, name, notests, parser.ParseComments|parser.Mode(0)) if err != nil { errorf("%s", err) return @@ -84,20 +84,23 @@ func doDir(name string) { } type Package struct { - p *ast.Package - fs *token.FileSet - decl map[string]ast.Node - used map[string]bool + p *ast.Package + fs *token.FileSet + decl map[string]ast.Node + missingcomments map[string]ast.Node + used map[string]bool } func doPackage(fs *token.FileSet, pkg *ast.Package) { p := &Package{ - p: pkg, - fs: fs, - decl: make(map[string]ast.Node), - used: make(map[string]bool), + p: pkg, + fs: fs, + decl: make(map[string]ast.Node), + missingcomments: make(map[string]ast.Node), + used: make(map[string]bool), } for _, file := range pkg.Files { + cmap := ast.NewCommentMap(fs, file, file.Comments) for _, decl := range file.Decls { switch n := decl.(type) { case *ast.GenDecl: @@ -115,6 +118,20 @@ func doPackage(fs *token.FileSet, pkg *ast.Package) { } } case *ast.FuncDecl: + // Skip if function is 'main' + if n.Name.Name == "main" { + continue + } + // Skip non-exported functions + if !ast.IsExported(n.Name.Name) { + continue + } + // Check if comments are missing from exported functions + _, ok := cmap[n] + if ok == false { + p.missingcomments[n.Name.Name] = n + } + // function declarations // TODO(remy): do methods if n.Recv == nil { @@ -140,6 +157,7 @@ func doPackage(fs *token.FileSet, pkg *ast.Package) { // walk file looking for used nodes. ast.Walk(p, file) } + // reports. reports := Reports(nil) for name, node := range p.decl { @@ -151,6 +169,10 @@ func doPackage(fs *token.FileSet, pkg *ast.Package) { for _, report := range reports { errorf("%s: %s is unused", fs.Position(report.pos), report.name) } + + for name, node := range p.missingcomments { + errorf("%s: comment is missing for 'func %s'", fs.Position(node.Pos()), name) + } } type Report struct { From 137584d658c4542e71db027d08ec4f187d69b883 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 3 Mar 2015 01:25:45 -0800 Subject: [PATCH 2/3] Add comments --- buildscripts/deadcode.go | 10 ++++-- pkg/storage/fs/fs.go | 11 +++++- pkg/storage/inmemory/inmemory.go | 49 ++++++++++++++++---------- pkg/utils/config/config_test.go | 3 +- pkg/utils/cpu/cpu.go | 3 ++ pkg/utils/crypto/sha512/sha512.go | 13 +++++-- pkg/utils/crypto/sha512/sha512_gen.go | 2 ++ pkg/utils/crypto/x509/generator.go | 1 + pkg/utils/helpers/common.go | 50 --------------------------- pkg/utils/policy/date.go | 1 + 10 files changed, 66 insertions(+), 77 deletions(-) delete mode 100644 pkg/utils/helpers/common.go diff --git a/buildscripts/deadcode.go b/buildscripts/deadcode.go index 76327a33e..479589413 100644 --- a/buildscripts/deadcode.go +++ b/buildscripts/deadcode.go @@ -118,15 +118,19 @@ func doPackage(fs *token.FileSet, pkg *ast.Package) { } } case *ast.FuncDecl: - // Skip if function is 'main' + // if function is 'main', never check if n.Name.Name == "main" { continue } - // Skip non-exported functions + // Do not be strict on non-exported functions if !ast.IsExported(n.Name.Name) { continue } - // Check if comments are missing from exported functions + // Do not be strict for field list functions + // if n.Recv != nil { + // continue + //} + // Be strict for global functions _, ok := cmap[n] if ok == false { p.missingcomments[n.Name.Name] = n diff --git a/pkg/storage/fs/fs.go b/pkg/storage/fs/fs.go index 892b1aa44..5b7c5e526 100644 --- a/pkg/storage/fs/fs.go +++ b/pkg/storage/fs/fs.go @@ -58,6 +58,15 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, s *storage) { close(errorChannel) } +func appendUniq(slice []string, i string) []string { + for _, ele := range slice { + if ele == i { + return slice + } + } + return append(slice, i) +} + // Bucket Operations func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) { files, err := ioutil.ReadDir(storage.root) @@ -379,7 +388,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso } metadataList = append(metadataList, metadata) case delimited != "": - resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited) + resources.CommonPrefixes = helpers.appendUniq(resources.CommonPrefixes, delimited) } case resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix): delimited := delimiter(name, resources.Delimiter) diff --git a/pkg/storage/inmemory/inmemory.go b/pkg/storage/inmemory/inmemory.go index 69cad2ef4..f30a6bbd8 100644 --- a/pkg/storage/inmemory/inmemory.go +++ b/pkg/storage/inmemory/inmemory.go @@ -47,6 +47,23 @@ type storedObject struct { data []byte } +// Start inmemory object server +func Start() (chan<- string, <-chan error, *storage) { + ctrlChannel := make(chan string) + errorChannel := make(chan error) + go start(ctrlChannel, errorChannel) + return ctrlChannel, errorChannel, &storage{ + bucketdata: make(map[string]storedBucket), + objectdata: make(map[string]storedObject), + lock: new(sync.RWMutex), + } +} + +func start(ctrlChannel <-chan string, errorChannel chan<- error) { + close(errorChannel) +} + +// GET object from memory buffer func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // TODO synchronize access // get object @@ -60,14 +77,17 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st } } +// Not implemented func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error { return mstorage.ApiNotImplemented{Api: "PutBucketPolicy"} } +// Not implemented func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) { return policy.BucketPolicy{}, mstorage.ApiNotImplemented{Api: "GetBucketPolicy"} } +// PUT object to memory buffer func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error { storage.lock.Lock() defer storage.lock.Unlock() @@ -108,6 +128,7 @@ func (storage *storage) StoreObject(bucket, key, contentType string, data io.Rea return nil } +// Create Bucket in memory func (storage *storage) StoreBucket(bucketName string) error { storage.lock.Lock() defer storage.lock.Unlock() @@ -128,11 +149,11 @@ func (storage *storage) StoreBucket(bucketName string) error { return nil } +// List objects in memory func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) { if _, ok := storage.bucketdata[bucket]; ok == false { return []mstorage.ObjectMetadata{}, mstorage.BucketResourcesMetadata{IsTruncated: false}, mstorage.BucketNotFound{Bucket: bucket} } - // TODO prefix and count handling var results []mstorage.ObjectMetadata var keys []string for key := range storage.objectdata { @@ -157,10 +178,16 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso type ByBucketName []mstorage.BucketMetadata -func (b ByBucketName) Len() int { return len(b) } -func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +// Len of bucket name +func (b ByBucketName) Len() int { return len(b) } + +// Swap bucket i, j +func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } + +// Less func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name } +// List buckets func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) { var results []mstorage.BucketMetadata for _, bucket := range storage.bucketdata { @@ -170,21 +197,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) { return results, nil } -func Start() (chan<- string, <-chan error, *storage) { - ctrlChannel := make(chan string) - errorChannel := make(chan error) - go start(ctrlChannel, errorChannel) - return ctrlChannel, errorChannel, &storage{ - bucketdata: make(map[string]storedBucket), - objectdata: make(map[string]storedObject), - lock: new(sync.RWMutex), - } -} - -func start(ctrlChannel <-chan string, errorChannel chan<- error) { - close(errorChannel) -} - +// HEAD object func (storage *storage) GetObjectMetadata(bucket, key string) (mstorage.ObjectMetadata, error) { objectKey := bucket + ":" + key diff --git a/pkg/utils/config/config_test.go b/pkg/utils/config/config_test.go index 71684fe1d..9513ce094 100644 --- a/pkg/utils/config/config_test.go +++ b/pkg/utils/config/config_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/minio-io/minio/pkg/utils/crypto/keys" - "github.com/minio-io/minio/pkg/utils/helpers" . "gopkg.in/check.v1" ) @@ -35,7 +34,7 @@ func Test(t *testing.T) { TestingT(t) } func (s *MySuite) TestConfig(c *C) { conf := Config{} - conf.configPath, _ = helpers.MakeTempTestDir() + conf.configPath, _ = ioutil.TempDir("/tmp", "minio-test-") defer os.RemoveAll(conf.configPath) conf.configFile = path.Join(conf.configPath, "config.json") if _, err := os.Stat(conf.configFile); os.IsNotExist(err) { diff --git a/pkg/utils/cpu/cpu.go b/pkg/utils/cpu/cpu.go index 8d552bfaa..eb3c58d99 100644 --- a/pkg/utils/cpu/cpu.go +++ b/pkg/utils/cpu/cpu.go @@ -21,14 +21,17 @@ package cpu // int has_avx2 (void); import "C" +// CPUID instruction verification wrapper for SSE41 extensions func HasSSE41() bool { return int(C.has_sse41()) == 1 } +// CPUID instruction verification wrapper for AVX extensions func HasAVX() bool { return int(C.has_avx()) == 1 } +// CPUID instruction verification wrapper for AVX2 extensions func HasAVX2() bool { return int(C.has_avx2()) == 1 } diff --git a/pkg/utils/crypto/sha512/sha512.go b/pkg/utils/crypto/sha512/sha512.go index 24d37f603..691826c56 100644 --- a/pkg/utils/crypto/sha512/sha512.go +++ b/pkg/utils/crypto/sha512/sha512.go @@ -55,6 +55,7 @@ func block(dig *digest, p []byte) { } } +// Reset digest to its default value func (d *digest) Reset() { d.h[0] = init0 d.h[1] = init1 @@ -75,12 +76,13 @@ func New() hash.Hash { return d } -func (d *digest) Size() int { - return Size -} +// Return output array byte size +func (d *digest) Size() int { return Size } +// Return blockSize func (d *digest) BlockSize() int { return BlockSize } +// Write blocks func (d *digest) Write(p []byte) (nn int, err error) { nn = len(p) d.len += uint64(nn) @@ -104,6 +106,7 @@ func (d *digest) Write(p []byte) (nn int, err error) { return } +// Calculate sha512 func (d0 *digest) Sum(in []byte) []byte { // Make a copy of d0 so that caller can keep writing and summing. d := new(digest) @@ -112,6 +115,7 @@ func (d0 *digest) Sum(in []byte) []byte { return append(in, hash[:]...) } +// internal checksum calculation, returns [Size]byte func (d *digest) checkSum() [Size]byte { // Padding. Add a 1 bit and 0 bits until 112 bytes mod 128. len := d.len @@ -153,6 +157,7 @@ func (d *digest) checkSum() [Size]byte { // Convenience functions +// Single caller function returns [Size]byte func Sum512(data []byte) [Size]byte { var d digest d.Reset() @@ -160,6 +165,7 @@ func Sum512(data []byte) [Size]byte { return d.checkSum() } +// Takes in io.Reader, low memory footprint checksum func Sum(reader io.Reader) ([]byte, error) { h := New() var err error @@ -176,6 +182,7 @@ func Sum(reader io.Reader) ([]byte, error) { return h.Sum(nil), nil } +// Similar to 'Sum()' but returns a [Size]byte func SumStream(reader io.Reader) ([Size]byte, error) { var returnValue [Size]byte sumSlice, err := Sum(reader) diff --git a/pkg/utils/crypto/sha512/sha512_gen.go b/pkg/utils/crypto/sha512/sha512_gen.go index 9641e19f0..e6fd8a424 100644 --- a/pkg/utils/crypto/sha512/sha512_gen.go +++ b/pkg/utils/crypto/sha512/sha512_gen.go @@ -29,6 +29,7 @@ import ( sha512intel "github.com/minio-io/minio/pkg/utils/crypto/sha512" ) +// Intels processor accelerated sha512 implementation func SumIntel(reader io.Reader) ([]byte, error) { h := sha512intel.New() var err error @@ -45,6 +46,7 @@ func SumIntel(reader io.Reader) ([]byte, error) { return h.Sum(nil), nil } +// Golang default implementation func Sum(reader io.Reader) ([]byte, error) { k := sha512.New() var err error diff --git a/pkg/utils/crypto/x509/generator.go b/pkg/utils/crypto/x509/generator.go index 6219e346c..650135b3f 100644 --- a/pkg/utils/crypto/x509/generator.go +++ b/pkg/utils/crypto/x509/generator.go @@ -72,6 +72,7 @@ func pemBlockForKey(priv interface{}) *pem.Block { } } +// Generate certificates using custom parameters func (tls *Certificates) GenerateCertificates(params X509Params) error { var rsaBits int = 2048 var priv interface{} diff --git a/pkg/utils/helpers/common.go b/pkg/utils/helpers/common.go deleted file mode 100644 index 242f55df0..000000000 --- a/pkg/utils/helpers/common.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Mini Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package helpers - -import ( - "io/ioutil" - "strings" -) - -// Create a new temp directory -func MakeTempTestDir() (string, error) { - return ioutil.TempDir("/tmp", "minio-test-") -} - -// Camelcase input string -func FirstUpper(str string) string { - return strings.ToUpper(str[0:1]) + str[1:] -} - -func AppendUniqInt(slice []int, i int) []int { - for _, ele := range slice { - if ele == i { - return slice - } - } - return append(slice, i) -} - -func AppendUniqStr(slice []string, i string) []string { - for _, ele := range slice { - if ele == i { - return slice - } - } - return append(slice, i) -} diff --git a/pkg/utils/policy/date.go b/pkg/utils/policy/date.go index b22e8207a..7fe6f04bb 100644 --- a/pkg/utils/policy/date.go +++ b/pkg/utils/policy/date.go @@ -13,6 +13,7 @@ type Date struct { Day byte } +// Date to string output in yyyy-mm-dd format func (d Date) String() string { return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day) } From 3a3c8645fc5125d7dd99ab8e2298e68958e94466 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 3 Mar 2015 02:36:12 -0800 Subject: [PATCH 3/3] Update comments across the codebase --- .gitignore | 2 +- Makefile | 9 +- buildscripts/{deadcode.go => verifier.go} | 181 +++++++++++---------- pkg/api/minioapi/generic_handlers.go | 5 +- pkg/api/minioapi/response.go | 9 +- pkg/api/webuiapi/webuiapi.go | 1 + pkg/server/httpserver/httpserver.go | 1 + pkg/server/server.go | 1 + pkg/storage/donut/data/data_v1/data.go | 3 + pkg/storage/donut/frame/frame_v1/donut.go | 1 + pkg/storage/fs/fs.go | 28 +++- pkg/storage/storage.go | 2 + pkg/storage/storage_api_suite.go | 1 + pkg/storage/storage_errors.go | 25 ++- pkg/utils/checksum/crc32c/crc32c_darwin.go | 4 + pkg/utils/checksum/crc32c/crc32c_linux.go | 10 +- pkg/utils/config/config_test.go | 1 + pkg/utils/crypto/md5/md5.go | 1 + pkg/utils/crypto/sha1/sha1.go | 10 +- pkg/utils/crypto/sha1/sha1_gen.go | 65 -------- pkg/utils/crypto/sha256/sha256.go | 14 +- pkg/utils/crypto/sha256/sha256_gen.go | 81 --------- pkg/utils/crypto/sha512/sha512.go | 6 +- pkg/utils/crypto/sha512/sha512_gen.go | 83 ---------- 24 files changed, 198 insertions(+), 346 deletions(-) rename buildscripts/{deadcode.go => verifier.go} (92%) delete mode 100644 pkg/utils/crypto/sha1/sha1_gen.go delete mode 100644 pkg/utils/crypto/sha256/sha256_gen.go delete mode 100644 pkg/utils/crypto/sha512/sha512_gen.go diff --git a/.gitignore b/.gitignore index f369d7cc4..930d00291 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ cover.out minio **/*.test **/*.sublime-workspace -deadcode \ No newline at end of file +verifier \ No newline at end of file diff --git a/Makefile b/Makefile index 039c225bc..071407f57 100644 --- a/Makefile +++ b/Makefile @@ -7,16 +7,17 @@ checkdeps: @(env bash $(PWD)/buildscripts/checkdeps.sh) checkgopath: - @echo "Checking project in ${MINIOPATH}" + @echo "Checking if project is at ${MINIOPATH}" @if [ ! -d ${MINIOPATH} ]; then echo "Project not found in $GOPATH, please follow instructions provided at https://github.com/Minio-io/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi getdeps: checkdeps checkgopath @go get github.com/tools/godep && echo "Installed godep" -deadcode: getdeps - @go run buildscripts/deadcode.go . +verifier: getdeps + @echo "Checking for offending code" + @go run buildscripts/verifier.go ${PWD} -build-all: getdeps deadcode +build-all: verifier @echo "Building Libraries" @godep go generate ./... @godep go build ./... diff --git a/buildscripts/deadcode.go b/buildscripts/verifier.go similarity index 92% rename from buildscripts/deadcode.go rename to buildscripts/verifier.go index 479589413..a987e12fe 100644 --- a/buildscripts/deadcode.go +++ b/buildscripts/verifier.go @@ -24,46 +24,91 @@ func appendUniq(slice []string, i string) []string { return append(slice, i) } -func getAllFiles(path string, fl os.FileInfo, err error) error { +// error formats the error to standard error, adding program +// identification and a newline +func errorf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "verifier: "+format+"\n", args...) + exitCode = 2 +} + +type Package struct { + p *ast.Package + fs *token.FileSet + decl map[string]ast.Node + missingcomments map[string]ast.Node + used map[string]bool +} + +type usedWalker Package + +// Walks through the AST marking used identifiers. +func (p *usedWalker) Visit(node ast.Node) ast.Visitor { + // just be stupid and mark all *ast.Ident + switch n := node.(type) { + case *ast.Ident: + p.used[n.Name] = true + } + return p +} + +type Report struct { + pos token.Pos + name string +} +type Reports []Report + +// Len +func (l Reports) Len() int { return len(l) } + +// Less +func (l Reports) Less(i, j int) bool { return l[i].pos < l[j].pos } + +// Swap +func (l Reports) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +// Visits files for used nodes. +func (p *Package) Visit(node ast.Node) ast.Visitor { + u := usedWalker(*p) // hopefully p fields are references. + switch n := node.(type) { + // don't walk whole file, but only: + case *ast.ValueSpec: + // - variable initializers + for _, value := range n.Values { + ast.Walk(&u, value) + } + // variable types. + if n.Type != nil { + ast.Walk(&u, n.Type) + } + case *ast.BlockStmt: + // - function bodies + for _, stmt := range n.List { + ast.Walk(&u, stmt) + } + case *ast.FuncDecl: + // - function signatures + ast.Walk(&u, n.Type) + case *ast.TypeSpec: + // - type declarations + ast.Walk(&u, n.Type) + } + return p +} + +func getAllMinioPkgs(path string, fl os.FileInfo, err error) error { if err != nil { return err } if fl.IsDir() { + // Skip godeps + if strings.Contains(path, "Godeps") { + return nil + } dirs = appendUniq(dirs, path) } return nil } -func main() { - flag.Parse() - if flag.NArg() == 0 { - doDir(".") - } else { - for _, name := range flag.Args() { - // Is it a directory? - if fi, err := os.Stat(name); err == nil && fi.IsDir() { - err := filepath.Walk(name, getAllFiles) - if err != nil { - errorf(err.Error()) - } - for _, dir := range dirs { - doDir(dir) - } - } else { - errorf("not a directory: %s", name) - } - } - } - os.Exit(exitCode) -} - -// error formats the error to standard error, adding program -// identification and a newline -func errorf(format string, args ...interface{}) { - fmt.Fprintf(os.Stderr, "deadcode: "+format+"\n", args...) - exitCode = 2 -} - func doDir(name string) { notests := func(info os.FileInfo) bool { if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && @@ -83,14 +128,6 @@ func doDir(name string) { } } -type Package struct { - p *ast.Package - fs *token.FileSet - decl map[string]ast.Node - missingcomments map[string]ast.Node - used map[string]bool -} - func doPackage(fs *token.FileSet, pkg *ast.Package) { p := &Package{ p: pkg, @@ -179,53 +216,25 @@ func doPackage(fs *token.FileSet, pkg *ast.Package) { } } -type Report struct { - pos token.Pos - name string -} -type Reports []Report - -func (l Reports) Len() int { return len(l) } -func (l Reports) Less(i, j int) bool { return l[i].pos < l[j].pos } -func (l Reports) Swap(i, j int) { l[i], l[j] = l[j], l[i] } - -// Visits files for used nodes. -func (p *Package) Visit(node ast.Node) ast.Visitor { - u := usedWalker(*p) // hopefully p fields are references. - switch n := node.(type) { - // don't walk whole file, but only: - case *ast.ValueSpec: - // - variable initializers - for _, value := range n.Values { - ast.Walk(&u, value) +func main() { + flag.Parse() + if flag.NArg() == 0 { + doDir(".") + } else { + for _, name := range flag.Args() { + // Is it a directory? + if fi, err := os.Stat(name); err == nil && fi.IsDir() { + err := filepath.Walk(name, getAllMinioPkgs) + if err != nil { + errorf(err.Error()) + } + for _, dir := range dirs { + doDir(dir) + } + } else { + errorf("not a directory: %s", name) + } } - // variable types. - if n.Type != nil { - ast.Walk(&u, n.Type) - } - case *ast.BlockStmt: - // - function bodies - for _, stmt := range n.List { - ast.Walk(&u, stmt) - } - case *ast.FuncDecl: - // - function signatures - ast.Walk(&u, n.Type) - case *ast.TypeSpec: - // - type declarations - ast.Walk(&u, n.Type) } - return p -} - -type usedWalker Package - -// Walks through the AST marking used identifiers. -func (p *usedWalker) Visit(node ast.Node) ast.Visitor { - // just be stupid and mark all *ast.Ident - switch n := node.(type) { - case *ast.Ident: - p.used[n.Name] = true - } - return p + os.Exit(exitCode) } diff --git a/pkg/api/minioapi/generic_handlers.go b/pkg/api/minioapi/generic_handlers.go index 623b965b5..f8512630d 100644 --- a/pkg/api/minioapi/generic_handlers.go +++ b/pkg/api/minioapi/generic_handlers.go @@ -52,6 +52,7 @@ func validateHandler(conf config.Config, h http.Handler) http.Handler { return vHandler{conf, h} } +// Validate handler ServeHTTP() wrapper func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { accessKey := stripAccessKey(r) acceptsContentType := getContentType(r) @@ -95,6 +96,7 @@ func ignoreResourcesHandler(h http.Handler) http.Handler { return rHandler{h} } +// Resource handler ServeHTTP() wrapper func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { acceptsContentType := getContentType(r) if ignoreUnImplementedObjectResources(r) || ignoreUnImplementedBucketResources(r) { @@ -109,7 +111,7 @@ func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { //// helpers -// Checks requests for unimplemented resources +// Checks requests for unimplemented Bucket resources func ignoreUnImplementedBucketResources(req *http.Request) bool { q := req.URL.Query() for name := range q { @@ -120,6 +122,7 @@ func ignoreUnImplementedBucketResources(req *http.Request) bool { return false } +// Checks requests for unimplemented Object resources func ignoreUnImplementedObjectResources(req *http.Request) bool { q := req.URL.Query() for name := range q { diff --git a/pkg/api/minioapi/response.go b/pkg/api/minioapi/response.go index 37cae1080..d5c806e3d 100644 --- a/pkg/api/minioapi/response.go +++ b/pkg/api/minioapi/response.go @@ -56,8 +56,13 @@ func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResp type ItemKey []*Item -func (b ItemKey) Len() int { return len(b) } -func (b ItemKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +// Len +func (b ItemKey) Len() int { return len(b) } + +// Swap +func (b ItemKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } + +// Less func (b ItemKey) Less(i, j int) bool { return b[i].Key < b[j].Key } // takes a set of objects and prepares the objects for serialization diff --git a/pkg/api/webuiapi/webuiapi.go b/pkg/api/webuiapi/webuiapi.go index 29b0675e8..c59418a70 100644 --- a/pkg/api/webuiapi/webuiapi.go +++ b/pkg/api/webuiapi/webuiapi.go @@ -42,6 +42,7 @@ type encoder interface { Encode(v interface{}) error } +// Http wrapper handler func HttpHandler() http.Handler { mux := mux.NewRouter() var api = webUiApi{} diff --git a/pkg/server/httpserver/httpserver.go b/pkg/server/httpserver/httpserver.go index 59dd404a5..5a82c826f 100644 --- a/pkg/server/httpserver/httpserver.go +++ b/pkg/server/httpserver/httpserver.go @@ -32,6 +32,7 @@ type HttpServerConfig struct { type HttpServer struct{} +// Start http server func Start(handler http.Handler, config HttpServerConfig) (chan<- string, <-chan error, *HttpServer) { ctrlChannel := make(chan string) errorChannel := make(chan error) diff --git a/pkg/server/server.go b/pkg/server/server.go index 2a09c7985..01a744c5e 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -143,6 +143,7 @@ func getStorageChannels(storageType StorageType) (ctrlChans []chan<- string, sta return } +// Create channels func Start(configs []ServerConfig) { // reflected looping is necessary to remove dead channels from loop and not flood switch ctrlChans, statusChans := getHttpChannels(configs) diff --git a/pkg/storage/donut/data/data_v1/data.go b/pkg/storage/donut/data/data_v1/data.go index 3e4e7b269..1556770c3 100644 --- a/pkg/storage/donut/data/data_v1/data.go +++ b/pkg/storage/donut/data/data_v1/data.go @@ -47,6 +47,7 @@ type EncoderParams struct { Technique EncoderTechnique } +// populate new header func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderParams) DataHeader { header := DataHeader{} header.Key = key @@ -61,6 +62,7 @@ func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderP return header } +// validate populated header func ValidateHeader(header DataHeader) bool { if header.Key == "" || header.Part < 0 || len(header.Metadata) < 2 { return false @@ -73,6 +75,7 @@ func ValidateHeader(header DataHeader) bool { return true } +// Write data, returns error upon any failure func WriteData(target io.Writer, header DataHeader, data io.Reader) error { if !ValidateHeader(header) { return fmt.Errorf("Invalid header") diff --git a/pkg/storage/donut/frame/frame_v1/donut.go b/pkg/storage/donut/frame/frame_v1/donut.go index fde680f33..d2cc1cb89 100644 --- a/pkg/storage/donut/frame/frame_v1/donut.go +++ b/pkg/storage/donut/frame/frame_v1/donut.go @@ -73,6 +73,7 @@ type DonutFrameFooter struct { type Data bytes.Buffer +// Write Donut format to input io.Writer, returns error upon any failure func WriteFrame(target io.Writer, reader io.Reader, length uint64) error { // write header header := DonutFrameHeader{ diff --git a/pkg/storage/fs/fs.go b/pkg/storage/fs/fs.go index 5b7c5e526..0e67046ac 100644 --- a/pkg/storage/fs/fs.go +++ b/pkg/storage/fs/fs.go @@ -29,7 +29,6 @@ import ( "sync" mstorage "github.com/minio-io/minio/pkg/storage" - "github.com/minio-io/minio/pkg/utils/helpers" "github.com/minio-io/minio/pkg/utils/policy" ) @@ -42,6 +41,7 @@ type SerializedMetadata struct { ContentType string } +// Start filesystem channel func Start(root string) (chan<- string, <-chan error, *storage) { ctrlChannel := make(chan string) errorChannel := make(chan error) @@ -67,7 +67,9 @@ func appendUniq(slice []string, i string) []string { return append(slice, i) } -// Bucket Operations +/// Bucket Operations + +// GET - Service func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) { files, err := ioutil.ReadDir(storage.root) if err != nil { @@ -92,6 +94,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) { return metadataList, nil } +// PUT - Bucket func (storage *storage) StoreBucket(bucket string) error { storage.lock.Lock() defer storage.lock.Unlock() @@ -119,6 +122,7 @@ func (storage *storage) StoreBucket(bucket string) error { return nil } +// GET - Bucket policy func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) { storage.lock.Lock() defer storage.lock.Unlock() @@ -163,6 +167,7 @@ func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) { } +// PUT - Bucket policy func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error { storage.lock.Lock() defer storage.lock.Unlock() @@ -203,8 +208,9 @@ func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) err return nil } -// Object Operations +/// Object Operations +// GET Object func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) { // validate bucket if mstorage.IsValidBucket(bucket) == false { @@ -247,6 +253,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st return count, nil } +// HEAD Object func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) { if mstorage.IsValidBucket(bucket) == false { return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket} @@ -340,10 +347,16 @@ func delimiter(path, delimiter string) string { type ByObjectKey []mstorage.ObjectMetadata -func (b ByObjectKey) Len() int { return len(b) } -func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +// Len +func (b ByObjectKey) Len() int { return len(b) } + +// Swap +func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } + +// Less func (b ByObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key } +// GET bucket (list objects) func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) { p := Path{} p.files = make(map[string]os.FileInfo) @@ -388,7 +401,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso } metadataList = append(metadataList, metadata) case delimited != "": - resources.CommonPrefixes = helpers.appendUniq(resources.CommonPrefixes, delimited) + resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited) } case resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix): delimited := delimiter(name, resources.Delimiter) @@ -403,7 +416,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso } metadataList = append(metadataList, metadata) case delimited != "": - resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited) + resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited) } case strings.HasPrefix(name, resources.Prefix): metadata := mstorage.ObjectMetadata{ @@ -422,6 +435,7 @@ ret: return metadataList, resources, nil } +// PUT object func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error { // TODO Commits should stage then move instead of writing directly storage.lock.Lock() diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 01fd041aa..cadace782 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -67,6 +67,7 @@ type BucketResourcesMetadata struct { Notification string } +// Verify Bucket name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html func IsValidBucket(bucket string) bool { if len(bucket) < 3 || len(bucket) > 63 { return false @@ -82,6 +83,7 @@ func IsValidBucket(bucket string) bool { return match } +// Verify Object name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func IsValidObject(object string) bool { if len(object) > 1024 || len(object) == 0 { return false diff --git a/pkg/storage/storage_api_suite.go b/pkg/storage/storage_api_suite.go index be38db8c6..e0c99f03b 100644 --- a/pkg/storage/storage_api_suite.go +++ b/pkg/storage/storage_api_suite.go @@ -24,6 +24,7 @@ import ( . "gopkg.in/check.v1" ) +// API test suite func APITestSuite(c *C, create func() Storage) { testCreateBucket(c, create) testMultipleObjectCreation(c, create) diff --git a/pkg/storage/storage_errors.go b/pkg/storage/storage_errors.go index 37c41dddf..fe5ea1f89 100644 --- a/pkg/storage/storage_errors.go +++ b/pkg/storage/storage_errors.go @@ -51,6 +51,14 @@ type ImplementationError struct { Err error } +type BackendCorrupted BackendError +type BucketPolicyNotFound GenericBucketError +type BucketNameInvalid GenericBucketError +type BucketExists GenericBucketError +type BucketNotFound GenericBucketError +type ObjectNameInvalid GenericObjectError + +// Return string an error formatted as the given text func (self ImplementationError) Error() string { error := "" if self.Bucket != "" { @@ -63,6 +71,7 @@ func (self ImplementationError) Error() string { return error } +// Wrapper function for error object func EmbedError(bucket, object string, err error) ImplementationError { return ImplementationError{ Bucket: bucket, @@ -71,45 +80,47 @@ func EmbedError(bucket, object string, err error) ImplementationError { } } -type BackendCorrupted BackendError -type BucketPolicyNotFound GenericBucketError -type BucketNameInvalid GenericBucketError -type BucketExists GenericBucketError -type BucketNotFound GenericBucketError -type ObjectNameInvalid GenericObjectError - +// Return string an error formatted as the given text func (self BucketPolicyNotFound) Error() string { return "Bucket policy not found for: " + self.Bucket } +// Return string an error formatted as the given text func (self ObjectNotFound) Error() string { return "Object not Found: " + self.Bucket + "#" + self.Object } +// Return string an error formatted as the given text func (self ApiNotImplemented) Error() string { return "Api not implemented: " + self.Api } +// Return string an error formatted as the given text func (self ObjectExists) Error() string { return "Object exists: " + self.Bucket + "#" + self.Key } +// Return string an error formatted as the given text func (self BucketNameInvalid) Error() string { return "Bucket name invalid: " + self.Bucket } +// Return string an error formatted as the given text func (self BucketExists) Error() string { return "Bucket exists: " + self.Bucket } +// Return string an error formatted as the given text func (self BucketNotFound) Error() string { return "Bucket not Found: " + self.Bucket } +// Return string an error formatted as the given text func (self ObjectNameInvalid) Error() string { return "Object name invalid: " + self.Bucket + "#" + self.Object } +// Return string an error formatted as the given text func (self BackendCorrupted) Error() string { return "Backend corrupted: " + self.Path } diff --git a/pkg/utils/checksum/crc32c/crc32c_darwin.go b/pkg/utils/checksum/crc32c/crc32c_darwin.go index 8e2599f35..11d20a522 100644 --- a/pkg/utils/checksum/crc32c/crc32c_darwin.go +++ b/pkg/utils/checksum/crc32c/crc32c_darwin.go @@ -23,6 +23,9 @@ import ( var castanagoliTable = crc32.MakeTable(crc32.Castagnoli) +/// Convenience functions + +// Single caller crc helper func Sum32(buffer []byte) uint32 { crc := crc32.New(castanagoliTable) crc.Reset() @@ -30,6 +33,7 @@ func Sum32(buffer []byte) uint32 { return crc.Sum32() } +// Low memory footprint io.Reader based crc helper func Sum(reader io.Reader) (uint32, error) { h := New() var err error diff --git a/pkg/utils/checksum/crc32c/crc32c_linux.go b/pkg/utils/checksum/crc32c/crc32c_linux.go index a1fde9149..fc234bd98 100644 --- a/pkg/utils/checksum/crc32c/crc32c_linux.go +++ b/pkg/utils/checksum/crc32c/crc32c_linux.go @@ -26,17 +26,22 @@ func New() hash.Hash32 { return &digest{crc: 0} } +// Return size of crc func (d *digest) Size() int { return Size } +// Stub func (d *digest) BlockSize() int { return 1 } +// Get crc in bytes func (d *digest) Sum(in []byte) []byte { s := d.crc return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) } +// Return current crc in digest object func (d *digest) Sum32() uint32 { return d.crc } +// Reset default crc func (d *digest) Reset() { d.crc = 0 } // Update returns the result of adding the bytes in p to the crc. @@ -44,13 +49,15 @@ func (d *digest) update(crc uint32, p []byte) uint32 { return updateCastanagoliPCL(crc, p) } +// Write data func (d *digest) Write(p []byte) (n int, err error) { d.crc = d.update(d.crc, p) return len(p), nil } -// Convenience functions +/// Convenience functions +// Single caller crc helper func Sum32(data []byte) uint32 { crc32 := New() crc32.Reset() @@ -58,6 +65,7 @@ func Sum32(data []byte) uint32 { return crc32.Sum32() } +// Low memory footprint io.Reader based crc helper func Sum(reader io.Reader) (uint32, error) { h := New() var err error diff --git a/pkg/utils/config/config_test.go b/pkg/utils/config/config_test.go index 9513ce094..9178d4f04 100644 --- a/pkg/utils/config/config_test.go +++ b/pkg/utils/config/config_test.go @@ -17,6 +17,7 @@ package config import ( + "io/ioutil" "os" "path" "sync" diff --git a/pkg/utils/crypto/md5/md5.go b/pkg/utils/crypto/md5/md5.go index 263d105e8..883d98847 100644 --- a/pkg/utils/crypto/md5/md5.go +++ b/pkg/utils/crypto/md5/md5.go @@ -21,6 +21,7 @@ import ( "io" ) +// Low memory footprint io.Reader based md5sum helper func Sum(reader io.Reader) ([]byte, error) { hash := md5.New() var err error diff --git a/pkg/utils/crypto/sha1/sha1.go b/pkg/utils/crypto/sha1/sha1.go index 7158a4038..af6a15527 100644 --- a/pkg/utils/crypto/sha1/sha1.go +++ b/pkg/utils/crypto/sha1/sha1.go @@ -40,6 +40,7 @@ type digest struct { len uint64 } +// Reset digest func (d *digest) Reset() { d.h[0] = init0 d.h[1] = init1 @@ -68,10 +69,13 @@ func block(dig *digest, p []byte) { } } +// Return output size func (d *digest) Size() int { return Size } +// Return checksum blocksize func (d *digest) BlockSize() int { return BlockSize } +// Write to digest func (d *digest) Write(p []byte) (nn int, err error) { nn = len(p) d.len += uint64(nn) @@ -95,6 +99,7 @@ func (d *digest) Write(p []byte) (nn int, err error) { return } +// Return checksum bytes func (d0 *digest) Sum(in []byte) []byte { // Make a copy of d0 so that caller can keep writing and summing. d := *d0 @@ -102,6 +107,7 @@ func (d0 *digest) Sum(in []byte) []byte { return append(in, hash[:]...) } +// Intermediate checksum function func (d *digest) checkSum() [Size]byte { len := d.len // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. @@ -135,8 +141,9 @@ func (d *digest) checkSum() [Size]byte { return digest } -// Convenience functions +/// Convenience functions +// Single caller sha1 helper func Sum1(data []byte) [Size]byte { var d digest d.Reset() @@ -144,6 +151,7 @@ func Sum1(data []byte) [Size]byte { return d.checkSum() } +// io.Reader based streaming sha1 helper func Sum(reader io.Reader) ([]byte, error) { h := New() var err error diff --git a/pkg/utils/crypto/sha1/sha1_gen.go b/pkg/utils/crypto/sha1/sha1_gen.go deleted file mode 100644 index 6386d80e9..000000000 --- a/pkg/utils/crypto/sha1/sha1_gen.go +++ /dev/null @@ -1,65 +0,0 @@ -// +build ignore - -// -// Mini Object Storage, (C) 2015 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "crypto/sha1" - "encoding/hex" - "fmt" - "io" - "os" - "time" - - sha1intel "github.com/minio-io/minio/pkg/utils/crypto/sha1" -) - -func Sum(reader io.Reader) ([]byte, error) { - k := sha1.New() - var err error - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - k.Write(byteBuffer) - } - if err != io.EOF { - return nil, err - } - return k.Sum(nil), nil -} - -func main() { - fmt.Println("-- start") - - file1, _ := os.Open("filename1") - defer file1.Close() - stark := time.Now() - sum, _ := Sum(file1) - endk := time.Since(stark) - - file2, _ := os.Open("filename2") - defer file2.Close() - starth := time.Now() - sumAVX2, _ := sha1intel.Sum(file2) - endh := time.Since(starth) - - fmt.Println("std(", endk, ")", "avx2(", endh, ")") - fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumAVX2)) -} diff --git a/pkg/utils/crypto/sha256/sha256.go b/pkg/utils/crypto/sha256/sha256.go index 3f90143a7..669ab79d8 100644 --- a/pkg/utils/crypto/sha256/sha256.go +++ b/pkg/utils/crypto/sha256/sha256.go @@ -43,6 +43,7 @@ type digest struct { len uint64 } +// Reset digest back to default func (d *digest) Reset() { d.h[0] = init0 d.h[1] = init1 @@ -74,12 +75,13 @@ func New() hash.Hash { return d } -func (d *digest) Size() int { - return Size -} +// Return size of checksum +func (d *digest) Size() int { return Size } +// Return blocksize of checksum func (d *digest) BlockSize() int { return BlockSize } +// Write to digest func (d *digest) Write(p []byte) (nn int, err error) { nn = len(p) d.len += uint64(nn) @@ -103,6 +105,7 @@ func (d *digest) Write(p []byte) (nn int, err error) { return } +// Return sha256 sum in bytes func (d0 *digest) Sum(in []byte) []byte { // Make a copy of d0 so that caller can keep writing and summing. d := *d0 @@ -110,6 +113,7 @@ func (d0 *digest) Sum(in []byte) []byte { return append(in, hash[:]...) } +// Intermediate checksum function func (d *digest) checkSum() [Size]byte { len := d.len // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. @@ -145,8 +149,9 @@ func (d *digest) checkSum() [Size]byte { return digest } -// Convenience functions +/// Convenience functions +// Single caller sha256 helper func Sum256(data []byte) [Size]byte { var d digest d.Reset() @@ -154,6 +159,7 @@ func Sum256(data []byte) [Size]byte { return d.checkSum() } +// io.Reader based streaming sha256 helper func Sum(reader io.Reader) ([]byte, error) { h := New() var err error diff --git a/pkg/utils/crypto/sha256/sha256_gen.go b/pkg/utils/crypto/sha256/sha256_gen.go deleted file mode 100644 index 6c73bce72..000000000 --- a/pkg/utils/crypto/sha256/sha256_gen.go +++ /dev/null @@ -1,81 +0,0 @@ -// +build ignore - -// -// Mini Object Storage, (C) 2015 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "os" - "time" - - sha256intel "github.com/minio-io/minio/pkg/utils/crypto/sha256" -) - -func SumIntel(reader io.Reader) ([]byte, error) { - h := sha256intel.New() - var err error - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - h.Write(byteBuffer) - } - if err != io.EOF { - return nil, err - } - return h.Sum(nil), nil -} - -func Sum(reader io.Reader) ([]byte, error) { - k := sha256.New() - var err error - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - k.Write(byteBuffer) - } - if err != io.EOF { - return nil, err - } - return k.Sum(nil), nil -} - -func main() { - fmt.Println("-- start") - - file1, _ := os.Open("filename1") - defer file1.Close() - stark := time.Now() - sum, _ := Sum(file1) - endk := time.Since(stark) - - file2, _ := os.Open("filename2") - defer file2.Close() - starth := time.Now() - sumSSE, _ := SumIntel(file2) - endh := time.Since(starth) - - fmt.Println("std(", endk, ")", "ssse3(", endh, ")") - fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE)) -} diff --git a/pkg/utils/crypto/sha512/sha512.go b/pkg/utils/crypto/sha512/sha512.go index 691826c56..7b1810a9d 100644 --- a/pkg/utils/crypto/sha512/sha512.go +++ b/pkg/utils/crypto/sha512/sha512.go @@ -155,9 +155,9 @@ func (d *digest) checkSum() [Size]byte { return digest } -// Convenience functions +/// Convenience functions -// Single caller function returns [Size]byte +// Single caller sha512 helper func Sum512(data []byte) [Size]byte { var d digest d.Reset() @@ -165,7 +165,7 @@ func Sum512(data []byte) [Size]byte { return d.checkSum() } -// Takes in io.Reader, low memory footprint checksum +// io.Reader based streaming sha512 helper func Sum(reader io.Reader) ([]byte, error) { h := New() var err error diff --git a/pkg/utils/crypto/sha512/sha512_gen.go b/pkg/utils/crypto/sha512/sha512_gen.go deleted file mode 100644 index e6fd8a424..000000000 --- a/pkg/utils/crypto/sha512/sha512_gen.go +++ /dev/null @@ -1,83 +0,0 @@ -// +build ignore - -// -// Mini Object Storage, (C) 2015 Minio, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "crypto/sha512" - "encoding/hex" - "fmt" - "io" - "os" - "time" - - sha512intel "github.com/minio-io/minio/pkg/utils/crypto/sha512" -) - -// Intels processor accelerated sha512 implementation -func SumIntel(reader io.Reader) ([]byte, error) { - h := sha512intel.New() - var err error - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - h.Write(byteBuffer) - } - if err != io.EOF { - return nil, err - } - return h.Sum(nil), nil -} - -// Golang default implementation -func Sum(reader io.Reader) ([]byte, error) { - k := sha512.New() - var err error - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - k.Write(byteBuffer) - } - if err != io.EOF { - return nil, err - } - return k.Sum(nil), nil -} - -func main() { - fmt.Println("-- start") - - file1, _ := os.Open("filename1") - defer file1.Close() - stark := time.Now() - sum, _ := Sum(file1) - endk := time.Since(stark) - - file2, _ := os.Open("filename2") - defer file2.Close() - starth := time.Now() - sumSSE, _ := SumIntel(file2) - endh := time.Since(starth) - - fmt.Println("std(", endk, ")", "ssse3(", endh, ")") - fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE)) -}