Merge pull request #245 from harshavardhana/pr_out_force_exported_functions_to_have_comments

.gitignore

@@ -5,4 +5,4 @@ cover.out
 minio
 **/*.test
 **/*.sublime-workspace
-deadcode
+verifier

Makefile

@@ -7,16 +7,17 @@ checkdeps:
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
 checkgopath:
-	@echo "Checking project in ${MINIOPATH}"
+	@echo "Checking if project is at ${MINIOPATH}"
 	@if [ ! -d ${MINIOPATH} ]; then echo "Project not found in $GOPATH, please follow instructions provided at https://github.com/Minio-io/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi
 
 getdeps: checkdeps checkgopath
 	@go get github.com/tools/godep && echo "Installed godep"
 
-deadcode: getdeps
+verifier: getdeps
-	@go run buildscripts/deadcode.go .
+	@echo "Checking for offending code"
+	@go run buildscripts/verifier.go ${PWD}
 
-build-all: getdeps deadcode
+build-all: verifier
 	@echo "Building Libraries"
 	@godep go generate ./...
 	@godep go build ./...

buildscripts/verifier.go

@@ -24,133 +24,31 @@ func appendUniq(slice []string, i string) []string {
 	return append(slice, i)
 }
 
-func getAllFiles(path string, fl os.FileInfo, err error) error {
-	if err != nil {
-		return err
-	}
-	if fl.IsDir() {
-		dirs = appendUniq(dirs, path)
-	}
-	return nil
-}
-
-func main() {
-	flag.Parse()
-	if flag.NArg() == 0 {
-		doDir(".")
-	} else {
-		for _, name := range flag.Args() {
-			// Is it a directory?
-			if fi, err := os.Stat(name); err == nil && fi.IsDir() {
-				err := filepath.Walk(name, getAllFiles)
-				if err != nil {
-					errorf(err.Error())
-				}
-				for _, dir := range dirs {
-					doDir(dir)
-				}
-			} else {
-				errorf("not a directory: %s", name)
-			}
-		}
-	}
-	os.Exit(exitCode)
-}
-
 // error formats the error to standard error, adding program
 // identification and a newline
 func errorf(format string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, "deadcode: "+format+"\n", args...)
+	fmt.Fprintf(os.Stderr, "verifier: "+format+"\n", args...)
 	exitCode = 2
 }
 
-func doDir(name string) {
-	notests := func(info os.FileInfo) bool {
-		if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") &&
-			!strings.HasSuffix(info.Name(), "_test.go") {
-			return true
-		}
-		return false
-	}
-	fs := token.NewFileSet()
-	pkgs, err := parser.ParseDir(fs, name, notests, parser.Mode(0))
-	if err != nil {
-		errorf("%s", err)
-		return
-	}
-	for _, pkg := range pkgs {
-		doPackage(fs, pkg)
-	}
-}
-
 type Package struct {
-	p    *ast.Package
+	p               *ast.Package
-	fs   *token.FileSet
+	fs              *token.FileSet
-	decl map[string]ast.Node
+	decl            map[string]ast.Node
-	used map[string]bool
+	missingcomments map[string]ast.Node
+	used            map[string]bool
 }
 
-func doPackage(fs *token.FileSet, pkg *ast.Package) {
+type usedWalker Package
-	p := &Package{
+
-		p:    pkg,
+// Walks through the AST marking used identifiers.
-		fs:   fs,
+func (p *usedWalker) Visit(node ast.Node) ast.Visitor {
-		decl: make(map[string]ast.Node),
+	// just be stupid and mark all *ast.Ident
-		used: make(map[string]bool),
+	switch n := node.(type) {
-	}
+	case *ast.Ident:
-	for _, file := range pkg.Files {
+		p.used[n.Name] = true
-		for _, decl := range file.Decls {
-			switch n := decl.(type) {
-			case *ast.GenDecl:
-				// var, const, types
-				for _, spec := range n.Specs {
-					switch s := spec.(type) {
-					case *ast.ValueSpec:
-						// constants and variables.
-						for _, name := range s.Names {
-							p.decl[name.Name] = n
-						}
-					case *ast.TypeSpec:
-						// type definitions.
-						p.decl[s.Name.Name] = n
-					}
-				}
-			case *ast.FuncDecl:
-				// function declarations
-				// TODO(remy): do methods
-				if n.Recv == nil {
-					p.decl[n.Name.Name] = n
-				}
-			}
-		}
-	}
-	// init() is always used
-	p.used["init"] = true
-	if pkg.Name != "main" {
-		// exported names are marked used for non-main packages.
-		for name := range p.decl {
-			if ast.IsExported(name) {
-				p.used[name] = true
-			}
-		}
-	} else {
-		// in main programs, main() is called.
-		p.used["main"] = true
-	}
-	for _, file := range pkg.Files {
-		// walk file looking for used nodes.
-		ast.Walk(p, file)
-	}
-	// reports.
-	reports := Reports(nil)
-	for name, node := range p.decl {
-		if !p.used[name] {
-			reports = append(reports, Report{node.Pos(), name})
-		}
-	}
-	sort.Sort(reports)
-	for _, report := range reports {
-		errorf("%s: %s is unused", fs.Position(report.pos), report.name)
 	}
+	return p
 }
 
 type Report struct {
@@ -159,9 +57,14 @@ type Report struct {
 }
 type Reports []Report
 
-func (l Reports) Len() int           { return len(l) }
+// Len
+func (l Reports) Len() int { return len(l) }
+
+// Less
 func (l Reports) Less(i, j int) bool { return l[i].pos < l[j].pos }
-func (l Reports) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
+
+// Swap
+func (l Reports) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
 
 // Visits files for used nodes.
 func (p *Package) Visit(node ast.Node) ast.Visitor {
@@ -192,14 +95,146 @@ func (p *Package) Visit(node ast.Node) ast.Visitor {
 	return p
 }
 
-type usedWalker Package
+func getAllMinioPkgs(path string, fl os.FileInfo, err error) error {
-
+	if err != nil {
-// Walks through the AST marking used identifiers.
+		return err
-func (p *usedWalker) Visit(node ast.Node) ast.Visitor {
-	// just be stupid and mark all *ast.Ident
-	switch n := node.(type) {
-	case *ast.Ident:
-		p.used[n.Name] = true
 	}
-	return p
+	if fl.IsDir() {
+		// Skip godeps
+		if strings.Contains(path, "Godeps") {
+			return nil
+		}
+		dirs = appendUniq(dirs, path)
+	}
+	return nil
+}
+
+func doDir(name string) {
+	notests := func(info os.FileInfo) bool {
+		if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") &&
+			!strings.HasSuffix(info.Name(), "_test.go") {
+			return true
+		}
+		return false
+	}
+	fs := token.NewFileSet()
+	pkgs, err := parser.ParseDir(fs, name, notests, parser.ParseComments|parser.Mode(0))
+	if err != nil {
+		errorf("%s", err)
+		return
+	}
+	for _, pkg := range pkgs {
+		doPackage(fs, pkg)
+	}
+}
+
+func doPackage(fs *token.FileSet, pkg *ast.Package) {
+	p := &Package{
+		p:               pkg,
+		fs:              fs,
+		decl:            make(map[string]ast.Node),
+		missingcomments: make(map[string]ast.Node),
+		used:            make(map[string]bool),
+	}
+	for _, file := range pkg.Files {
+		cmap := ast.NewCommentMap(fs, file, file.Comments)
+		for _, decl := range file.Decls {
+			switch n := decl.(type) {
+			case *ast.GenDecl:
+				// var, const, types
+				for _, spec := range n.Specs {
+					switch s := spec.(type) {
+					case *ast.ValueSpec:
+						// constants and variables.
+						for _, name := range s.Names {
+							p.decl[name.Name] = n
+						}
+					case *ast.TypeSpec:
+						// type definitions.
+						p.decl[s.Name.Name] = n
+					}
+				}
+			case *ast.FuncDecl:
+				// if function is 'main', never check
+				if n.Name.Name == "main" {
+					continue
+				}
+				// Do not be strict on non-exported functions
+				if !ast.IsExported(n.Name.Name) {
+					continue
+				}
+				// Do not be strict for field list functions
+				// if n.Recv != nil {
+				// continue
+				//}
+				// Be strict for global functions
+				_, ok := cmap[n]
+				if ok == false {
+					p.missingcomments[n.Name.Name] = n
+				}
+
+				// function declarations
+				// TODO(remy): do methods
+				if n.Recv == nil {
+					p.decl[n.Name.Name] = n
+				}
+			}
+		}
+	}
+	// init() is always used
+	p.used["init"] = true
+	if pkg.Name != "main" {
+		// exported names are marked used for non-main packages.
+		for name := range p.decl {
+			if ast.IsExported(name) {
+				p.used[name] = true
+			}
+		}
+	} else {
+		// in main programs, main() is called.
+		p.used["main"] = true
+	}
+	for _, file := range pkg.Files {
+		// walk file looking for used nodes.
+		ast.Walk(p, file)
+	}
+
+	// reports.
+	reports := Reports(nil)
+	for name, node := range p.decl {
+		if !p.used[name] {
+			reports = append(reports, Report{node.Pos(), name})
+		}
+	}
+	sort.Sort(reports)
+	for _, report := range reports {
+		errorf("%s: %s is unused", fs.Position(report.pos), report.name)
+	}
+
+	for name, node := range p.missingcomments {
+		errorf("%s: comment is missing for 'func %s'", fs.Position(node.Pos()), name)
+	}
+}
+
+func main() {
+	flag.Parse()
+	if flag.NArg() == 0 {
+		doDir(".")
+	} else {
+		for _, name := range flag.Args() {
+			// Is it a directory?
+			if fi, err := os.Stat(name); err == nil && fi.IsDir() {
+				err := filepath.Walk(name, getAllMinioPkgs)
+				if err != nil {
+					errorf(err.Error())
+				}
+				for _, dir := range dirs {
+					doDir(dir)
+				}
+			} else {
+				errorf("not a directory: %s", name)
+			}
+		}
+	}
+	os.Exit(exitCode)
 }

pkg/api/minioapi/generic_handlers.go

@@ -52,6 +52,7 @@ func validateHandler(conf config.Config, h http.Handler) http.Handler {
 	return vHandler{conf, h}
 }
 
+// Validate handler ServeHTTP() wrapper
 func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	accessKey := stripAccessKey(r)
 	acceptsContentType := getContentType(r)
@@ -95,6 +96,7 @@ func ignoreResourcesHandler(h http.Handler) http.Handler {
 	return rHandler{h}
 }
 
+// Resource handler ServeHTTP() wrapper
 func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	acceptsContentType := getContentType(r)
 	if ignoreUnImplementedObjectResources(r) || ignoreUnImplementedBucketResources(r) {
@@ -109,7 +111,7 @@ func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 //// helpers
 
-// Checks requests for unimplemented resources
+// Checks requests for unimplemented Bucket resources
 func ignoreUnImplementedBucketResources(req *http.Request) bool {
 	q := req.URL.Query()
 	for name := range q {
@@ -120,6 +122,7 @@ func ignoreUnImplementedBucketResources(req *http.Request) bool {
 	return false
 }
 
+// Checks requests for unimplemented Object resources
 func ignoreUnImplementedObjectResources(req *http.Request) bool {
 	q := req.URL.Query()
 	for name := range q {

pkg/api/minioapi/response.go

@@ -56,8 +56,13 @@ func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResp
 
 type ItemKey []*Item
 
-func (b ItemKey) Len() int           { return len(b) }
+// Len
-func (b ItemKey) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b ItemKey) Len() int { return len(b) }
+
+// Swap
+func (b ItemKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+
+// Less
 func (b ItemKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
 
 // takes a set of objects and prepares the objects for serialization

pkg/api/webuiapi/webuiapi.go

@@ -42,6 +42,7 @@ type encoder interface {
 	Encode(v interface{}) error
 }
 
+// Http wrapper handler
 func HttpHandler() http.Handler {
 	mux := mux.NewRouter()
 	var api = webUiApi{}

pkg/server/httpserver/httpserver.go

@@ -32,6 +32,7 @@ type HttpServerConfig struct {
 
 type HttpServer struct{}
 
+// Start http server
 func Start(handler http.Handler, config HttpServerConfig) (chan<- string, <-chan error, *HttpServer) {
 	ctrlChannel := make(chan string)
 	errorChannel := make(chan error)

pkg/server/server.go

@@ -143,6 +143,7 @@ func getStorageChannels(storageType StorageType) (ctrlChans []chan<- string, sta
 	return
 }
 
+// Create channels
 func Start(configs []ServerConfig) {
 	// reflected looping is necessary to remove dead channels from loop and not flood switch
 	ctrlChans, statusChans := getHttpChannels(configs)

pkg/storage/donut/data/data_v1/data.go

@@ -47,6 +47,7 @@ type EncoderParams struct {
 	Technique EncoderTechnique
 }
 
+// populate new header
 func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderParams) DataHeader {
 	header := DataHeader{}
 	header.Key = key
@@ -61,6 +62,7 @@ func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderP
 	return header
 }
 
+// validate populated header
 func ValidateHeader(header DataHeader) bool {
 	if header.Key == "" || header.Part < 0 || len(header.Metadata) < 2 {
 		return false
@@ -73,6 +75,7 @@ func ValidateHeader(header DataHeader) bool {
 	return true
 }
 
+// Write data, returns error upon any failure
 func WriteData(target io.Writer, header DataHeader, data io.Reader) error {
 	if !ValidateHeader(header) {
 		return fmt.Errorf("Invalid header")

pkg/storage/donut/frame/frame_v1/donut.go

@@ -73,6 +73,7 @@ type DonutFrameFooter struct {
 
 type Data bytes.Buffer
 
+// Write Donut format to input io.Writer, returns error upon any failure
 func WriteFrame(target io.Writer, reader io.Reader, length uint64) error {
 	// write header
 	header := DonutFrameHeader{

pkg/storage/fs/fs.go

@@ -29,7 +29,6 @@ import (
 	"sync"
 
 	mstorage "github.com/minio-io/minio/pkg/storage"
-	"github.com/minio-io/minio/pkg/utils/helpers"
 	"github.com/minio-io/minio/pkg/utils/policy"
 )
 
@@ -42,6 +41,7 @@ type SerializedMetadata struct {
 	ContentType string
 }
 
+// Start filesystem channel
 func Start(root string) (chan<- string, <-chan error, *storage) {
 	ctrlChannel := make(chan string)
 	errorChannel := make(chan error)
@@ -58,7 +58,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, s *storage) {
 	close(errorChannel)
 }
 
-// Bucket Operations
+func appendUniq(slice []string, i string) []string {
+	for _, ele := range slice {
+		if ele == i {
+			return slice
+		}
+	}
+	return append(slice, i)
+}
+
+/// Bucket Operations
+
+// GET - Service
 func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
 	files, err := ioutil.ReadDir(storage.root)
 	if err != nil {
@@ -83,6 +94,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
 	return metadataList, nil
 }
 
+// PUT - Bucket
 func (storage *storage) StoreBucket(bucket string) error {
 	storage.lock.Lock()
 	defer storage.lock.Unlock()
@@ -110,6 +122,7 @@ func (storage *storage) StoreBucket(bucket string) error {
 	return nil
 }
 
+// GET - Bucket policy
 func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
 	storage.lock.Lock()
 	defer storage.lock.Unlock()
@@ -154,6 +167,7 @@ func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
 
 }
 
+// PUT - Bucket policy
 func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
 	storage.lock.Lock()
 	defer storage.lock.Unlock()
@@ -194,8 +208,9 @@ func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) err
 	return nil
 }
 
-// Object Operations
+/// Object Operations
 
+// GET Object
 func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
 	// validate bucket
 	if mstorage.IsValidBucket(bucket) == false {
@@ -238,6 +253,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
 	return count, nil
 }
 
+// HEAD Object
 func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) {
 	if mstorage.IsValidBucket(bucket) == false {
 		return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
@@ -331,10 +347,16 @@ func delimiter(path, delimiter string) string {
 
 type ByObjectKey []mstorage.ObjectMetadata
 
-func (b ByObjectKey) Len() int           { return len(b) }
+// Len
-func (b ByObjectKey) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b ByObjectKey) Len() int { return len(b) }
+
+// Swap
+func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+
+// Less
 func (b ByObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
 
+// GET bucket (list objects)
 func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
 	p := Path{}
 	p.files = make(map[string]os.FileInfo)
@@ -379,7 +401,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
 				}
 				metadataList = append(metadataList, metadata)
 			case delimited != "":
-				resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited)
+				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited)
 			}
 		case resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix):
 			delimited := delimiter(name, resources.Delimiter)
@@ -394,7 +416,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
 				}
 				metadataList = append(metadataList, metadata)
 			case delimited != "":
-				resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited)
+				resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited)
 			}
 		case strings.HasPrefix(name, resources.Prefix):
 			metadata := mstorage.ObjectMetadata{
@@ -413,6 +435,7 @@ ret:
 	return metadataList, resources, nil
 }
 
+// PUT object
 func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
 	// TODO Commits should stage then move instead of writing directly
 	storage.lock.Lock()

pkg/storage/inmemory/inmemory.go

@@ -47,6 +47,23 @@ type storedObject struct {
 	data     []byte
 }
 
+// Start inmemory object server
+func Start() (chan<- string, <-chan error, *storage) {
+	ctrlChannel := make(chan string)
+	errorChannel := make(chan error)
+	go start(ctrlChannel, errorChannel)
+	return ctrlChannel, errorChannel, &storage{
+		bucketdata: make(map[string]storedBucket),
+		objectdata: make(map[string]storedObject),
+		lock:       new(sync.RWMutex),
+	}
+}
+
+func start(ctrlChannel <-chan string, errorChannel chan<- error) {
+	close(errorChannel)
+}
+
+// GET object from memory buffer
 func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
 	// TODO synchronize access
 	// get object
@@ -60,14 +77,17 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
 	}
 }
 
+// Not implemented
 func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
 	return mstorage.ApiNotImplemented{Api: "PutBucketPolicy"}
 }
 
+// Not implemented
 func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
 	return policy.BucketPolicy{}, mstorage.ApiNotImplemented{Api: "GetBucketPolicy"}
 }
 
+// PUT object to memory buffer
 func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
 	storage.lock.Lock()
 	defer storage.lock.Unlock()
@@ -108,6 +128,7 @@ func (storage *storage) StoreObject(bucket, key, contentType string, data io.Rea
 	return nil
 }
 
+// Create Bucket in memory
 func (storage *storage) StoreBucket(bucketName string) error {
 	storage.lock.Lock()
 	defer storage.lock.Unlock()
@@ -128,11 +149,11 @@ func (storage *storage) StoreBucket(bucketName string) error {
 	return nil
 }
 
+// List objects in memory
 func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
 	if _, ok := storage.bucketdata[bucket]; ok == false {
 		return []mstorage.ObjectMetadata{}, mstorage.BucketResourcesMetadata{IsTruncated: false}, mstorage.BucketNotFound{Bucket: bucket}
 	}
-	// TODO prefix and count handling
 	var results []mstorage.ObjectMetadata
 	var keys []string
 	for key := range storage.objectdata {
@@ -157,10 +178,16 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
 
 type ByBucketName []mstorage.BucketMetadata
 
-func (b ByBucketName) Len() int           { return len(b) }
+// Len of bucket name
-func (b ByBucketName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b ByBucketName) Len() int { return len(b) }
+
+// Swap bucket i, j
+func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+
+// Less
 func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
 
+// List buckets
 func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
 	var results []mstorage.BucketMetadata
 	for _, bucket := range storage.bucketdata {
@@ -170,21 +197,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
 	return results, nil
 }
 
-func Start() (chan<- string, <-chan error, *storage) {
+// HEAD object
-	ctrlChannel := make(chan string)
-	errorChannel := make(chan error)
-	go start(ctrlChannel, errorChannel)
-	return ctrlChannel, errorChannel, &storage{
-		bucketdata: make(map[string]storedBucket),
-		objectdata: make(map[string]storedObject),
-		lock:       new(sync.RWMutex),
-	}
-}
-
-func start(ctrlChannel <-chan string, errorChannel chan<- error) {
-	close(errorChannel)
-}
-
 func (storage *storage) GetObjectMetadata(bucket, key string) (mstorage.ObjectMetadata, error) {
 	objectKey := bucket + ":" + key
 

pkg/storage/storage.go

@@ -67,6 +67,7 @@ type BucketResourcesMetadata struct {
 	Notification string
 }
 
+// Verify Bucket name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
 func IsValidBucket(bucket string) bool {
 	if len(bucket) < 3 || len(bucket) > 63 {
 		return false
@@ -82,6 +83,7 @@ func IsValidBucket(bucket string) bool {
 	return match
 }
 
+// Verify Object name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
 func IsValidObject(object string) bool {
 	if len(object) > 1024 || len(object) == 0 {
 		return false

pkg/storage/storage_api_suite.go

@@ -24,6 +24,7 @@ import (
 	. "gopkg.in/check.v1"
 )
 
+// API test suite
 func APITestSuite(c *C, create func() Storage) {
 	testCreateBucket(c, create)
 	testMultipleObjectCreation(c, create)

pkg/storage/storage_errors.go

@@ -51,6 +51,14 @@ type ImplementationError struct {
 	Err    error
 }
 
+type BackendCorrupted BackendError
+type BucketPolicyNotFound GenericBucketError
+type BucketNameInvalid GenericBucketError
+type BucketExists GenericBucketError
+type BucketNotFound GenericBucketError
+type ObjectNameInvalid GenericObjectError
+
+// Return string an error formatted as the given text
 func (self ImplementationError) Error() string {
 	error := ""
 	if self.Bucket != "" {
@@ -63,6 +71,7 @@ func (self ImplementationError) Error() string {
 	return error
 }
 
+// Wrapper function for error object
 func EmbedError(bucket, object string, err error) ImplementationError {
 	return ImplementationError{
 		Bucket: bucket,
@@ -71,45 +80,47 @@ func EmbedError(bucket, object string, err error) ImplementationError {
 	}
 }
 
-type BackendCorrupted BackendError
+// Return string an error formatted as the given text
-type BucketPolicyNotFound GenericBucketError
-type BucketNameInvalid GenericBucketError
-type BucketExists GenericBucketError
-type BucketNotFound GenericBucketError
-type ObjectNameInvalid GenericObjectError
-
 func (self BucketPolicyNotFound) Error() string {
 	return "Bucket policy not found for: " + self.Bucket
 }
 
+// Return string an error formatted as the given text
 func (self ObjectNotFound) Error() string {
 	return "Object not Found: " + self.Bucket + "#" + self.Object
 }
 
+// Return string an error formatted as the given text
 func (self ApiNotImplemented) Error() string {
 	return "Api not implemented: " + self.Api
 }
 
+// Return string an error formatted as the given text
 func (self ObjectExists) Error() string {
 	return "Object exists: " + self.Bucket + "#" + self.Key
 }
 
+// Return string an error formatted as the given text
 func (self BucketNameInvalid) Error() string {
 	return "Bucket name invalid: " + self.Bucket
 }
 
+// Return string an error formatted as the given text
 func (self BucketExists) Error() string {
 	return "Bucket exists: " + self.Bucket
 }
 
+// Return string an error formatted as the given text
 func (self BucketNotFound) Error() string {
 	return "Bucket not Found: " + self.Bucket
 }
 
+// Return string an error formatted as the given text
 func (self ObjectNameInvalid) Error() string {
 	return "Object name invalid: " + self.Bucket + "#" + self.Object
 }
 
+// Return string an error formatted as the given text
 func (self BackendCorrupted) Error() string {
 	return "Backend corrupted: " + self.Path
 }

pkg/utils/checksum/crc32c/crc32c_darwin.go

@@ -23,6 +23,9 @@ import (
 
 var castanagoliTable = crc32.MakeTable(crc32.Castagnoli)
 
+/// Convenience functions
+
+// Single caller crc helper
 func Sum32(buffer []byte) uint32 {
 	crc := crc32.New(castanagoliTable)
 	crc.Reset()
@@ -30,6 +33,7 @@ func Sum32(buffer []byte) uint32 {
 	return crc.Sum32()
 }
 
+// Low memory footprint io.Reader based crc helper
 func Sum(reader io.Reader) (uint32, error) {
 	h := New()
 	var err error

pkg/utils/checksum/crc32c/crc32c_linux.go

@@ -26,17 +26,22 @@ func New() hash.Hash32 {
 	return &digest{crc: 0}
 }
 
+// Return size of crc
 func (d *digest) Size() int { return Size }
 
+// Stub
 func (d *digest) BlockSize() int { return 1 }
 
+// Get crc in bytes
 func (d *digest) Sum(in []byte) []byte {
 	s := d.crc
 	return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
 }
 
+// Return current crc in digest object
 func (d *digest) Sum32() uint32 { return d.crc }
 
+// Reset default crc
 func (d *digest) Reset() { d.crc = 0 }
 
 // Update returns the result of adding the bytes in p to the crc.
@@ -44,13 +49,15 @@ func (d *digest) update(crc uint32, p []byte) uint32 {
 	return updateCastanagoliPCL(crc, p)
 }
 
+// Write data
 func (d *digest) Write(p []byte) (n int, err error) {
 	d.crc = d.update(d.crc, p)
 	return len(p), nil
 }
 
-// Convenience functions
+/// Convenience functions
 
+// Single caller crc helper
 func Sum32(data []byte) uint32 {
 	crc32 := New()
 	crc32.Reset()
@@ -58,6 +65,7 @@ func Sum32(data []byte) uint32 {
 	return crc32.Sum32()
 }
 
+// Low memory footprint io.Reader based crc helper
 func Sum(reader io.Reader) (uint32, error) {
 	h := New()
 	var err error

pkg/utils/config/config_test.go

@@ -17,13 +17,13 @@
 package config
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
 	"sync"
 	"testing"
 
 	"github.com/minio-io/minio/pkg/utils/crypto/keys"
-	"github.com/minio-io/minio/pkg/utils/helpers"
 	. "gopkg.in/check.v1"
 )
 
@@ -35,7 +35,7 @@ func Test(t *testing.T) { TestingT(t) }
 
 func (s *MySuite) TestConfig(c *C) {
 	conf := Config{}
-	conf.configPath, _ = helpers.MakeTempTestDir()
+	conf.configPath, _ = ioutil.TempDir("/tmp", "minio-test-")
 	defer os.RemoveAll(conf.configPath)
 	conf.configFile = path.Join(conf.configPath, "config.json")
 	if _, err := os.Stat(conf.configFile); os.IsNotExist(err) {

pkg/utils/cpu/cpu.go

@@ -21,14 +21,17 @@ package cpu
 // int has_avx2 (void);
 import "C"
 
+// CPUID instruction verification wrapper for SSE41 extensions
 func HasSSE41() bool {
 	return int(C.has_sse41()) == 1
 }
 
+// CPUID instruction verification wrapper for AVX extensions
 func HasAVX() bool {
 	return int(C.has_avx()) == 1
 }
 
+// CPUID instruction verification wrapper for AVX2 extensions
 func HasAVX2() bool {
 	return int(C.has_avx2()) == 1
 }

pkg/utils/crypto/md5/md5.go

@@ -21,6 +21,7 @@ import (
 	"io"
 )
 
+// Low memory footprint io.Reader based md5sum helper
 func Sum(reader io.Reader) ([]byte, error) {
 	hash := md5.New()
 	var err error

pkg/utils/crypto/sha1/sha1.go

@@ -40,6 +40,7 @@ type digest struct {
 	len uint64
 }
 
+// Reset digest
 func (d *digest) Reset() {
 	d.h[0] = init0
 	d.h[1] = init1
@@ -68,10 +69,13 @@ func block(dig *digest, p []byte) {
 	}
 }
 
+// Return output size
 func (d *digest) Size() int { return Size }
 
+// Return checksum blocksize
 func (d *digest) BlockSize() int { return BlockSize }
 
+// Write to digest
 func (d *digest) Write(p []byte) (nn int, err error) {
 	nn = len(p)
 	d.len += uint64(nn)
@@ -95,6 +99,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
 	return
 }
 
+// Return checksum bytes
 func (d0 *digest) Sum(in []byte) []byte {
 	// Make a copy of d0 so that caller can keep writing and summing.
 	d := *d0
@@ -102,6 +107,7 @@ func (d0 *digest) Sum(in []byte) []byte {
 	return append(in, hash[:]...)
 }
 
+// Intermediate checksum function
 func (d *digest) checkSum() [Size]byte {
 	len := d.len
 	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
@@ -135,8 +141,9 @@ func (d *digest) checkSum() [Size]byte {
 	return digest
 }
 
-// Convenience functions
+/// Convenience functions
 
+// Single caller sha1 helper
 func Sum1(data []byte) [Size]byte {
 	var d digest
 	d.Reset()
@@ -144,6 +151,7 @@ func Sum1(data []byte) [Size]byte {
 	return d.checkSum()
 }
 
+// io.Reader based streaming sha1 helper
 func Sum(reader io.Reader) ([]byte, error) {
 	h := New()
 	var err error

pkg/utils/crypto/sha1/sha1_gen.go

@@ -1,65 +0,0 @@
-// +build ignore
-
-//
-// Mini Object Storage, (C) 2015 Minio, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package main
-
-import (
-	"crypto/sha1"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"os"
-	"time"
-
-	sha1intel "github.com/minio-io/minio/pkg/utils/crypto/sha1"
-)
-
-func Sum(reader io.Reader) ([]byte, error) {
-	k := sha1.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		k.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return k.Sum(nil), nil
-}
-
-func main() {
-	fmt.Println("-- start")
-
-	file1, _ := os.Open("filename1")
-	defer file1.Close()
-	stark := time.Now()
-	sum, _ := Sum(file1)
-	endk := time.Since(stark)
-
-	file2, _ := os.Open("filename2")
-	defer file2.Close()
-	starth := time.Now()
-	sumAVX2, _ := sha1intel.Sum(file2)
-	endh := time.Since(starth)
-
-	fmt.Println("std(", endk, ")", "avx2(", endh, ")")
-	fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumAVX2))
-}

pkg/utils/crypto/sha256/sha256.go

@@ -43,6 +43,7 @@ type digest struct {
 	len uint64
 }
 
+// Reset digest back to default
 func (d *digest) Reset() {
 	d.h[0] = init0
 	d.h[1] = init1
@@ -74,12 +75,13 @@ func New() hash.Hash {
 	return d
 }
 
-func (d *digest) Size() int {
+// Return size of checksum
-	return Size
+func (d *digest) Size() int { return Size }
-}
 
+// Return blocksize of checksum
 func (d *digest) BlockSize() int { return BlockSize }
 
+// Write to digest
 func (d *digest) Write(p []byte) (nn int, err error) {
 	nn = len(p)
 	d.len += uint64(nn)
@@ -103,6 +105,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
 	return
 }
 
+// Return sha256 sum in bytes
 func (d0 *digest) Sum(in []byte) []byte {
 	// Make a copy of d0 so that caller can keep writing and summing.
 	d := *d0
@@ -110,6 +113,7 @@ func (d0 *digest) Sum(in []byte) []byte {
 	return append(in, hash[:]...)
 }
 
+// Intermediate checksum function
 func (d *digest) checkSum() [Size]byte {
 	len := d.len
 	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
@@ -145,8 +149,9 @@ func (d *digest) checkSum() [Size]byte {
 	return digest
 }
 
-// Convenience functions
+/// Convenience functions
 
+// Single caller sha256 helper
 func Sum256(data []byte) [Size]byte {
 	var d digest
 	d.Reset()
@@ -154,6 +159,7 @@ func Sum256(data []byte) [Size]byte {
 	return d.checkSum()
 }
 
+// io.Reader based streaming sha256 helper
 func Sum(reader io.Reader) ([]byte, error) {
 	h := New()
 	var err error

pkg/utils/crypto/sha256/sha256_gen.go

@@ -1,81 +0,0 @@
-// +build ignore
-
-//
-// Mini Object Storage, (C) 2015 Minio, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package main
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"os"
-	"time"
-
-	sha256intel "github.com/minio-io/minio/pkg/utils/crypto/sha256"
-)
-
-func SumIntel(reader io.Reader) ([]byte, error) {
-	h := sha256intel.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		h.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return h.Sum(nil), nil
-}
-
-func Sum(reader io.Reader) ([]byte, error) {
-	k := sha256.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		k.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return k.Sum(nil), nil
-}
-
-func main() {
-	fmt.Println("-- start")
-
-	file1, _ := os.Open("filename1")
-	defer file1.Close()
-	stark := time.Now()
-	sum, _ := Sum(file1)
-	endk := time.Since(stark)
-
-	file2, _ := os.Open("filename2")
-	defer file2.Close()
-	starth := time.Now()
-	sumSSE, _ := SumIntel(file2)
-	endh := time.Since(starth)
-
-	fmt.Println("std(", endk, ")", "ssse3(", endh, ")")
-	fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE))
-}

pkg/utils/crypto/sha512/sha512.go

@@ -55,6 +55,7 @@ func block(dig *digest, p []byte) {
 	}
 }
 
+// Reset digest to its default value
 func (d *digest) Reset() {
 	d.h[0] = init0
 	d.h[1] = init1
@@ -75,12 +76,13 @@ func New() hash.Hash {
 	return d
 }
 
-func (d *digest) Size() int {
+// Return output array byte size
-	return Size
+func (d *digest) Size() int { return Size }
-}
 
+// Return blockSize
 func (d *digest) BlockSize() int { return BlockSize }
 
+// Write blocks
 func (d *digest) Write(p []byte) (nn int, err error) {
 	nn = len(p)
 	d.len += uint64(nn)
@@ -104,6 +106,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
 	return
 }
 
+// Calculate sha512
 func (d0 *digest) Sum(in []byte) []byte {
 	// Make a copy of d0 so that caller can keep writing and summing.
 	d := new(digest)
@@ -112,6 +115,7 @@ func (d0 *digest) Sum(in []byte) []byte {
 	return append(in, hash[:]...)
 }
 
+// internal checksum calculation, returns [Size]byte
 func (d *digest) checkSum() [Size]byte {
 	// Padding.  Add a 1 bit and 0 bits until 112 bytes mod 128.
 	len := d.len
@@ -151,8 +155,9 @@ func (d *digest) checkSum() [Size]byte {
 	return digest
 }
 
-// Convenience functions
+/// Convenience functions
 
+// Single caller sha512 helper
 func Sum512(data []byte) [Size]byte {
 	var d digest
 	d.Reset()
@@ -160,6 +165,7 @@ func Sum512(data []byte) [Size]byte {
 	return d.checkSum()
 }
 
+// io.Reader based streaming sha512 helper
 func Sum(reader io.Reader) ([]byte, error) {
 	h := New()
 	var err error
@@ -176,6 +182,7 @@ func Sum(reader io.Reader) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// Similar to 'Sum()' but returns a [Size]byte
 func SumStream(reader io.Reader) ([Size]byte, error) {
 	var returnValue [Size]byte
 	sumSlice, err := Sum(reader)

pkg/utils/crypto/sha512/sha512_gen.go

@@ -1,81 +0,0 @@
-// +build ignore
-
-//
-// Mini Object Storage, (C) 2015 Minio, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package main
-
-import (
-	"crypto/sha512"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"os"
-	"time"
-
-	sha512intel "github.com/minio-io/minio/pkg/utils/crypto/sha512"
-)
-
-func SumIntel(reader io.Reader) ([]byte, error) {
-	h := sha512intel.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		h.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return h.Sum(nil), nil
-}
-
-func Sum(reader io.Reader) ([]byte, error) {
-	k := sha512.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		k.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return k.Sum(nil), nil
-}
-
-func main() {
-	fmt.Println("-- start")
-
-	file1, _ := os.Open("filename1")
-	defer file1.Close()
-	stark := time.Now()
-	sum, _ := Sum(file1)
-	endk := time.Since(stark)
-
-	file2, _ := os.Open("filename2")
-	defer file2.Close()
-	starth := time.Now()
-	sumSSE, _ := SumIntel(file2)
-	endh := time.Since(starth)
-
-	fmt.Println("std(", endk, ")", "ssse3(", endh, ")")
-	fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE))
-}

pkg/utils/crypto/x509/generator.go

@@ -72,6 +72,7 @@ func pemBlockForKey(priv interface{}) *pem.Block {
 	}
 }
 
+// Generate certificates using custom parameters
 func (tls *Certificates) GenerateCertificates(params X509Params) error {
 	var rsaBits int = 2048
 	var priv interface{}

pkg/utils/helpers/common.go

@@ -1,50 +0,0 @@
-/*
- * Mini Object Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package helpers
-
-import (
-	"io/ioutil"
-	"strings"
-)
-
-// Create a new temp directory
-func MakeTempTestDir() (string, error) {
-	return ioutil.TempDir("/tmp", "minio-test-")
-}
-
-// Camelcase input string
-func FirstUpper(str string) string {
-	return strings.ToUpper(str[0:1]) + str[1:]
-}
-
-func AppendUniqInt(slice []int, i int) []int {
-	for _, ele := range slice {
-		if ele == i {
-			return slice
-		}
-	}
-	return append(slice, i)
-}
-
-func AppendUniqStr(slice []string, i string) []string {
-	for _, ele := range slice {
-		if ele == i {
-			return slice
-		}
-	}
-	return append(slice, i)
-}

pkg/utils/policy/date.go

@@ -13,6 +13,7 @@ type Date struct {
 	Day   byte
 }
 
+// Date to string output in yyyy-mm-dd format
 func (d Date) String() string {
 	return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
 }