MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-24 13:13:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #245 from harshavardhana/pr_out_force_exported_functions_to_have_comments

Browse Source
This commit is contained in:
Harshavardhana 2015-03-03 02:41:24 -08:00
commit 3db32f4935
29 changed files with 320 additions and 457 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -5,4 +5,4 @@ cover.out
minio
**/*.test
**/*.sublime-workspace
deadcode
verifier

9
Makefile
View File

@ -7,16 +7,17 @@ checkdeps:
@(env bash $(PWD)/buildscripts/checkdeps.sh)
checkgopath:
@echo "Checking project in ${MINIOPATH}"
@echo "Checking if project is at ${MINIOPATH}"
@if [ ! -d ${MINIOPATH} ]; then echo "Project not found in $GOPATH, please follow instructions provided at https://github.com/Minio-io/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi
getdeps: checkdeps checkgopath
@go get github.com/tools/godep && echo "Installed godep"
deadcode: getdeps
@go run buildscripts/deadcode.go .
verifier: getdeps
@echo "Checking for offending code"
@go run buildscripts/verifier.go ${PWD}
build-all: getdeps deadcode
build-all: verifier
@echo "Building Libraries"
@godep go generate ./...
@godep go build ./...

291
buildscripts/deadcode.go → buildscripts/verifier.go
View File

@ -24,133 +24,31 @@ func appendUniq(slice []string, i string) []string {
return append(slice, i)
}
func getAllFiles(path string, fl os.FileInfo, err error) error {
if err != nil {
return err
}
if fl.IsDir() {
dirs = appendUniq(dirs, path)
}
return nil
}
func main() {
flag.Parse()
if flag.NArg() == 0 {
doDir(".")
} else {
for _, name := range flag.Args() {
// Is it a directory?
if fi, err := os.Stat(name); err == nil && fi.IsDir() {
err := filepath.Walk(name, getAllFiles)
if err != nil {
errorf(err.Error())
}
for _, dir := range dirs {
doDir(dir)
}
} else {
errorf("not a directory: %s", name)
}
}
}
os.Exit(exitCode)
}
// error formats the error to standard error, adding program
// identification and a newline
func errorf(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "deadcode: "+format+"\n", args...)
fmt.Fprintf(os.Stderr, "verifier: "+format+"\n", args...)
exitCode = 2
}
func doDir(name string) {
notests := func(info os.FileInfo) bool {
if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") &&
!strings.HasSuffix(info.Name(), "_test.go") {
return true
}
return false
}
fs := token.NewFileSet()
pkgs, err := parser.ParseDir(fs, name, notests, parser.Mode(0))
if err != nil {
errorf("%s", err)
return
}
for _, pkg := range pkgs {
doPackage(fs, pkg)
}
}
type Package struct {
p *ast.Package
fs *token.FileSet
decl map[string]ast.Node
used map[string]bool
p *ast.Package
fs *token.FileSet
decl map[string]ast.Node
missingcomments map[string]ast.Node
used map[string]bool
}
func doPackage(fs *token.FileSet, pkg *ast.Package) {
p := &Package{
p: pkg,
fs: fs,
decl: make(map[string]ast.Node),
used: make(map[string]bool),
}
for _, file := range pkg.Files {
for _, decl := range file.Decls {
switch n := decl.(type) {
case *ast.GenDecl:
// var, const, types
for _, spec := range n.Specs {
switch s := spec.(type) {
case *ast.ValueSpec:
// constants and variables.
for _, name := range s.Names {
p.decl[name.Name] = n
}
case *ast.TypeSpec:
// type definitions.
p.decl[s.Name.Name] = n
}
}
case *ast.FuncDecl:
// function declarations
// TODO(remy): do methods
if n.Recv == nil {
p.decl[n.Name.Name] = n
}
}
}
}
// init() is always used
p.used["init"] = true
if pkg.Name != "main" {
// exported names are marked used for non-main packages.
for name := range p.decl {
if ast.IsExported(name) {
p.used[name] = true
}
}
} else {
// in main programs, main() is called.
p.used["main"] = true
}
for _, file := range pkg.Files {
// walk file looking for used nodes.
ast.Walk(p, file)
}
// reports.
reports := Reports(nil)
for name, node := range p.decl {
if !p.used[name] {
reports = append(reports, Report{node.Pos(), name})
}
}
sort.Sort(reports)
for _, report := range reports {
errorf("%s: %s is unused", fs.Position(report.pos), report.name)
type usedWalker Package
// Walks through the AST marking used identifiers.
func (p *usedWalker) Visit(node ast.Node) ast.Visitor {
// just be stupid and mark all *ast.Ident
switch n := node.(type) {
case *ast.Ident:
p.used[n.Name] = true
}
return p
}
type Report struct {
@ -159,9 +57,14 @@ type Report struct {
}
type Reports []Report
func (l Reports) Len() int { return len(l) }
// Len
func (l Reports) Len() int { return len(l) }
// Less
func (l Reports) Less(i, j int) bool { return l[i].pos < l[j].pos }
func (l Reports) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
// Swap
func (l Reports) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
// Visits files for used nodes.
func (p *Package) Visit(node ast.Node) ast.Visitor {
@ -192,14 +95,146 @@ func (p *Package) Visit(node ast.Node) ast.Visitor {
return p
}
type usedWalker Package
// Walks through the AST marking used identifiers.
func (p *usedWalker) Visit(node ast.Node) ast.Visitor {
// just be stupid and mark all *ast.Ident
switch n := node.(type) {
case *ast.Ident:
p.used[n.Name] = true
func getAllMinioPkgs(path string, fl os.FileInfo, err error) error {
if err != nil {
return err
}
return p
if fl.IsDir() {
// Skip godeps
if strings.Contains(path, "Godeps") {
return nil
}
dirs = appendUniq(dirs, path)
}
return nil
}
func doDir(name string) {
notests := func(info os.FileInfo) bool {
if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") &&
!strings.HasSuffix(info.Name(), "_test.go") {
return true
}
return false
}
fs := token.NewFileSet()
pkgs, err := parser.ParseDir(fs, name, notests, parser.ParseComments|parser.Mode(0))
if err != nil {
errorf("%s", err)
return
}
for _, pkg := range pkgs {
doPackage(fs, pkg)
}
}
func doPackage(fs *token.FileSet, pkg *ast.Package) {
p := &Package{
p: pkg,
fs: fs,
decl: make(map[string]ast.Node),
missingcomments: make(map[string]ast.Node),
used: make(map[string]bool),
}
for _, file := range pkg.Files {
cmap := ast.NewCommentMap(fs, file, file.Comments)
for _, decl := range file.Decls {
switch n := decl.(type) {
case *ast.GenDecl:
// var, const, types
for _, spec := range n.Specs {
switch s := spec.(type) {
case *ast.ValueSpec:
// constants and variables.
for _, name := range s.Names {
p.decl[name.Name] = n
}
case *ast.TypeSpec:
// type definitions.
p.decl[s.Name.Name] = n
}
}
case *ast.FuncDecl:
// if function is 'main', never check
if n.Name.Name == "main" {
continue
}
// Do not be strict on non-exported functions
if !ast.IsExported(n.Name.Name) {
continue
}
// Do not be strict for field list functions
// if n.Recv != nil {
// continue
//}
// Be strict for global functions
_, ok := cmap[n]
if ok == false {
p.missingcomments[n.Name.Name] = n
}
// function declarations
// TODO(remy): do methods
if n.Recv == nil {
p.decl[n.Name.Name] = n
}
}
}
}
// init() is always used
p.used["init"] = true
if pkg.Name != "main" {
// exported names are marked used for non-main packages.
for name := range p.decl {
if ast.IsExported(name) {
p.used[name] = true
}
}
} else {
// in main programs, main() is called.
p.used["main"] = true
}
for _, file := range pkg.Files {
// walk file looking for used nodes.
ast.Walk(p, file)
}
// reports.
reports := Reports(nil)
for name, node := range p.decl {
if !p.used[name] {
reports = append(reports, Report{node.Pos(), name})
}
}
sort.Sort(reports)
for _, report := range reports {
errorf("%s: %s is unused", fs.Position(report.pos), report.name)
}
for name, node := range p.missingcomments {
errorf("%s: comment is missing for 'func %s'", fs.Position(node.Pos()), name)
}
}
func main() {
flag.Parse()
if flag.NArg() == 0 {
doDir(".")
} else {
for _, name := range flag.Args() {
// Is it a directory?
if fi, err := os.Stat(name); err == nil && fi.IsDir() {
err := filepath.Walk(name, getAllMinioPkgs)
if err != nil {
errorf(err.Error())
}
for _, dir := range dirs {
doDir(dir)
}
} else {
errorf("not a directory: %s", name)
}
}
}
os.Exit(exitCode)
}

5
pkg/api/minioapi/generic_handlers.go
View File

@ -52,6 +52,7 @@ func validateHandler(conf config.Config, h http.Handler) http.Handler {
return vHandler{conf, h}
}
// Validate handler ServeHTTP() wrapper
func (h vHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
accessKey := stripAccessKey(r)
acceptsContentType := getContentType(r)
@ -95,6 +96,7 @@ func ignoreResourcesHandler(h http.Handler) http.Handler {
return rHandler{h}
}
// Resource handler ServeHTTP() wrapper
func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
acceptsContentType := getContentType(r)
if ignoreUnImplementedObjectResources(r) || ignoreUnImplementedBucketResources(r) {
@ -109,7 +111,7 @@ func (h rHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
//// helpers
// Checks requests for unimplemented resources
// Checks requests for unimplemented Bucket resources
func ignoreUnImplementedBucketResources(req *http.Request) bool {
q := req.URL.Query()
for name := range q {
@ -120,6 +122,7 @@ func ignoreUnImplementedBucketResources(req *http.Request) bool {
return false
}
// Checks requests for unimplemented Object resources
func ignoreUnImplementedObjectResources(req *http.Request) bool {
q := req.URL.Query()
for name := range q {

9
pkg/api/minioapi/response.go
View File

@ -56,8 +56,13 @@ func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResp
type ItemKey []*Item
func (b ItemKey) Len() int { return len(b) }
func (b ItemKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Len
func (b ItemKey) Len() int { return len(b) }
// Swap
func (b ItemKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b ItemKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
// takes a set of objects and prepares the objects for serialization

1
pkg/api/webuiapi/webuiapi.go
View File

@ -42,6 +42,7 @@ type encoder interface {
Encode(v interface{}) error
}
// Http wrapper handler
func HttpHandler() http.Handler {
mux := mux.NewRouter()
var api = webUiApi{}

1
pkg/server/httpserver/httpserver.go
View File

@ -32,6 +32,7 @@ type HttpServerConfig struct {
type HttpServer struct{}
// Start http server
func Start(handler http.Handler, config HttpServerConfig) (chan<- string, <-chan error, *HttpServer) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)

1
pkg/server/server.go
View File

@ -143,6 +143,7 @@ func getStorageChannels(storageType StorageType) (ctrlChans []chan<- string, sta
return
}
// Create channels
func Start(configs []ServerConfig) {
// reflected looping is necessary to remove dead channels from loop and not flood switch
ctrlChans, statusChans := getHttpChannels(configs)

3
pkg/storage/donut/data/data_v1/data.go
View File

@ -47,6 +47,7 @@ type EncoderParams struct {
Technique EncoderTechnique
}
// populate new header
func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderParams) DataHeader {
header := DataHeader{}
header.Key = key
@ -61,6 +62,7 @@ func NewHeader(key string, part uint8, metadata Metadata, encoderParams EncoderP
return header
}
// validate populated header
func ValidateHeader(header DataHeader) bool {
if header.Key == "" || header.Part < 0 || len(header.Metadata) < 2 {
return false
@ -73,6 +75,7 @@ func ValidateHeader(header DataHeader) bool {
return true
}
// Write data, returns error upon any failure
func WriteData(target io.Writer, header DataHeader, data io.Reader) error {
if !ValidateHeader(header) {
return fmt.Errorf("Invalid header")

1
pkg/storage/donut/frame/frame_v1/donut.go
View File

@ -73,6 +73,7 @@ type DonutFrameFooter struct {
type Data bytes.Buffer
// Write Donut format to input io.Writer, returns error upon any failure
func WriteFrame(target io.Writer, reader io.Reader, length uint64) error {
// write header
header := DonutFrameHeader{

37
pkg/storage/fs/fs.go
View File

@ -29,7 +29,6 @@ import (
"sync"
mstorage "github.com/minio-io/minio/pkg/storage"
"github.com/minio-io/minio/pkg/utils/helpers"
"github.com/minio-io/minio/pkg/utils/policy"
)
@ -42,6 +41,7 @@ type SerializedMetadata struct {
ContentType string
}
// Start filesystem channel
func Start(root string) (chan<- string, <-chan error, *storage) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
@ -58,7 +58,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error, s *storage) {
close(errorChannel)
}
// Bucket Operations
func appendUniq(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
/// Bucket Operations
// GET - Service
func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
files, err := ioutil.ReadDir(storage.root)
if err != nil {
@ -83,6 +94,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
return metadataList, nil
}
// PUT - Bucket
func (storage *storage) StoreBucket(bucket string) error {
storage.lock.Lock()
defer storage.lock.Unlock()
@ -110,6 +122,7 @@ func (storage *storage) StoreBucket(bucket string) error {
return nil
}
// GET - Bucket policy
func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
storage.lock.Lock()
defer storage.lock.Unlock()
@ -154,6 +167,7 @@ func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
}
// PUT - Bucket policy
func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
storage.lock.Lock()
defer storage.lock.Unlock()
@ -194,8 +208,9 @@ func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) err
return nil
}
// Object Operations
/// Object Operations
// GET Object
func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
// validate bucket
if mstorage.IsValidBucket(bucket) == false {
@ -238,6 +253,7 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
return count, nil
}
// HEAD Object
func (storage *storage) GetObjectMetadata(bucket string, object string) (mstorage.ObjectMetadata, error) {
if mstorage.IsValidBucket(bucket) == false {
return mstorage.ObjectMetadata{}, mstorage.BucketNameInvalid{Bucket: bucket}
@ -331,10 +347,16 @@ func delimiter(path, delimiter string) string {
type ByObjectKey []mstorage.ObjectMetadata
func (b ByObjectKey) Len() int { return len(b) }
func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Len
func (b ByObjectKey) Len() int { return len(b) }
// Swap
func (b ByObjectKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b ByObjectKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
// GET bucket (list objects)
func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
p := Path{}
p.files = make(map[string]os.FileInfo)
@ -379,7 +401,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
}
metadataList = append(metadataList, metadata)
case delimited != "":
resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited)
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited)
}
case resources.Delimiter != "" && strings.HasPrefix(name, resources.Prefix):
delimited := delimiter(name, resources.Delimiter)
@ -394,7 +416,7 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
}
metadataList = append(metadataList, metadata)
case delimited != "":
resources.CommonPrefixes = helpers.AppendUniqStr(resources.CommonPrefixes, delimited)
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimited)
}
case strings.HasPrefix(name, resources.Prefix):
metadata := mstorage.ObjectMetadata{
@ -413,6 +435,7 @@ ret:
return metadataList, resources, nil
}
// PUT object
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
// TODO Commits should stage then move instead of writing directly
storage.lock.Lock()

49
pkg/storage/inmemory/inmemory.go
View File

@ -47,6 +47,23 @@ type storedObject struct {
data []byte
}
// Start inmemory object server
func Start() (chan<- string, <-chan error, *storage) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, &storage{
bucketdata: make(map[string]storedBucket),
objectdata: make(map[string]storedObject),
lock: new(sync.RWMutex),
}
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
close(errorChannel)
}
// GET object from memory buffer
func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object string) (int64, error) {
// TODO synchronize access
// get object
@ -60,14 +77,17 @@ func (storage *storage) CopyObjectToWriter(w io.Writer, bucket string, object st
}
}
// Not implemented
func (storage *storage) StoreBucketPolicy(bucket string, policy interface{}) error {
return mstorage.ApiNotImplemented{Api: "PutBucketPolicy"}
}
// Not implemented
func (storage *storage) GetBucketPolicy(bucket string) (interface{}, error) {
return policy.BucketPolicy{}, mstorage.ApiNotImplemented{Api: "GetBucketPolicy"}
}
// PUT object to memory buffer
func (storage *storage) StoreObject(bucket, key, contentType string, data io.Reader) error {
storage.lock.Lock()
defer storage.lock.Unlock()
@ -108,6 +128,7 @@ func (storage *storage) StoreObject(bucket, key, contentType string, data io.Rea
return nil
}
// Create Bucket in memory
func (storage *storage) StoreBucket(bucketName string) error {
storage.lock.Lock()
defer storage.lock.Unlock()
@ -128,11 +149,11 @@ func (storage *storage) StoreBucket(bucketName string) error {
return nil
}
// List objects in memory
func (storage *storage) ListObjects(bucket string, resources mstorage.BucketResourcesMetadata) ([]mstorage.ObjectMetadata, mstorage.BucketResourcesMetadata, error) {
if _, ok := storage.bucketdata[bucket]; ok == false {
return []mstorage.ObjectMetadata{}, mstorage.BucketResourcesMetadata{IsTruncated: false}, mstorage.BucketNotFound{Bucket: bucket}
}
// TODO prefix and count handling
var results []mstorage.ObjectMetadata
var keys []string
for key := range storage.objectdata {
@ -157,10 +178,16 @@ func (storage *storage) ListObjects(bucket string, resources mstorage.BucketReso
type ByBucketName []mstorage.BucketMetadata
func (b ByBucketName) Len() int { return len(b) }
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Len of bucket name
func (b ByBucketName) Len() int { return len(b) }
// Swap bucket i, j
func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// Less
func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name }
// List buckets
func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
var results []mstorage.BucketMetadata
for _, bucket := range storage.bucketdata {
@ -170,21 +197,7 @@ func (storage *storage) ListBuckets() ([]mstorage.BucketMetadata, error) {
return results, nil
}
func Start() (chan<- string, <-chan error, *storage) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, &storage{
bucketdata: make(map[string]storedBucket),
objectdata: make(map[string]storedObject),
lock: new(sync.RWMutex),
}
}
func start(ctrlChannel <-chan string, errorChannel chan<- error) {
close(errorChannel)
}
// HEAD object
func (storage *storage) GetObjectMetadata(bucket, key string) (mstorage.ObjectMetadata, error) {
objectKey := bucket + ":" + key

2
pkg/storage/storage.go
View File

@ -67,6 +67,7 @@ type BucketResourcesMetadata struct {
Notification string
}
// Verify Bucket name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
func IsValidBucket(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
@ -82,6 +83,7 @@ func IsValidBucket(bucket string) bool {
return match
}
// Verify Object name in accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
func IsValidObject(object string) bool {
if len(object) > 1024 || len(object) == 0 {
return false

1
pkg/storage/storage_api_suite.go
View File

@ -24,6 +24,7 @@ import (
. "gopkg.in/check.v1"
)
// API test suite
func APITestSuite(c *C, create func() Storage) {
testCreateBucket(c, create)
testMultipleObjectCreation(c, create)

25
pkg/storage/storage_errors.go
View File

@ -51,6 +51,14 @@ type ImplementationError struct {
Err error
}
type BackendCorrupted BackendError
type BucketPolicyNotFound GenericBucketError
type BucketNameInvalid GenericBucketError
type BucketExists GenericBucketError
type BucketNotFound GenericBucketError
type ObjectNameInvalid GenericObjectError
// Return string an error formatted as the given text
func (self ImplementationError) Error() string {
error := ""
if self.Bucket != "" {
@ -63,6 +71,7 @@ func (self ImplementationError) Error() string {
return error
}
// Wrapper function for error object
func EmbedError(bucket, object string, err error) ImplementationError {
return ImplementationError{
Bucket: bucket,
@ -71,45 +80,47 @@ func EmbedError(bucket, object string, err error) ImplementationError {
}
}
type BackendCorrupted BackendError
type BucketPolicyNotFound GenericBucketError
type BucketNameInvalid GenericBucketError
type BucketExists GenericBucketError
type BucketNotFound GenericBucketError
type ObjectNameInvalid GenericObjectError
// Return string an error formatted as the given text
func (self BucketPolicyNotFound) Error() string {
return "Bucket policy not found for: " + self.Bucket
}
// Return string an error formatted as the given text
func (self ObjectNotFound) Error() string {
return "Object not Found: " + self.Bucket + "#" + self.Object
}
// Return string an error formatted as the given text
func (self ApiNotImplemented) Error() string {
return "Api not implemented: " + self.Api
}
// Return string an error formatted as the given text
func (self ObjectExists) Error() string {
return "Object exists: " + self.Bucket + "#" + self.Key
}
// Return string an error formatted as the given text
func (self BucketNameInvalid) Error() string {
return "Bucket name invalid: " + self.Bucket
}
// Return string an error formatted as the given text
func (self BucketExists) Error() string {
return "Bucket exists: " + self.Bucket
}
// Return string an error formatted as the given text
func (self BucketNotFound) Error() string {
return "Bucket not Found: " + self.Bucket
}
// Return string an error formatted as the given text
func (self ObjectNameInvalid) Error() string {
return "Object name invalid: " + self.Bucket + "#" + self.Object
}
// Return string an error formatted as the given text
func (self BackendCorrupted) Error() string {
return "Backend corrupted: " + self.Path
}

4
pkg/utils/checksum/crc32c/crc32c_darwin.go
View File

@ -23,6 +23,9 @@ import (
var castanagoliTable = crc32.MakeTable(crc32.Castagnoli)
/// Convenience functions
// Single caller crc helper
func Sum32(buffer []byte) uint32 {
crc := crc32.New(castanagoliTable)
crc.Reset()
@ -30,6 +33,7 @@ func Sum32(buffer []byte) uint32 {
return crc.Sum32()
}
// Low memory footprint io.Reader based crc helper
func Sum(reader io.Reader) (uint32, error) {
h := New()
var err error

10
pkg/utils/checksum/crc32c/crc32c_linux.go
View File

@ -26,17 +26,22 @@ func New() hash.Hash32 {
return &digest{crc: 0}
}
// Return size of crc
func (d *digest) Size() int { return Size }
// Stub
func (d *digest) BlockSize() int { return 1 }
// Get crc in bytes
func (d *digest) Sum(in []byte) []byte {
s := d.crc
return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
}
// Return current crc in digest object
func (d *digest) Sum32() uint32 { return d.crc }
// Reset default crc
func (d *digest) Reset() { d.crc = 0 }
// Update returns the result of adding the bytes in p to the crc.
@ -44,13 +49,15 @@ func (d *digest) update(crc uint32, p []byte) uint32 {
return updateCastanagoliPCL(crc, p)
}
// Write data
func (d *digest) Write(p []byte) (n int, err error) {
d.crc = d.update(d.crc, p)
return len(p), nil
}
// Convenience functions
/// Convenience functions
// Single caller crc helper
func Sum32(data []byte) uint32 {
crc32 := New()
crc32.Reset()
@ -58,6 +65,7 @@ func Sum32(data []byte) uint32 {
return crc32.Sum32()
}
// Low memory footprint io.Reader based crc helper
func Sum(reader io.Reader) (uint32, error) {
h := New()
var err error

4
pkg/utils/config/config_test.go
View File

@ -17,13 +17,13 @@
package config
import (
"io/ioutil"
"os"
"path"
"sync"
"testing"
"github.com/minio-io/minio/pkg/utils/crypto/keys"
"github.com/minio-io/minio/pkg/utils/helpers"
. "gopkg.in/check.v1"
)
@ -35,7 +35,7 @@ func Test(t *testing.T) { TestingT(t) }
func (s *MySuite) TestConfig(c *C) {
conf := Config{}
conf.configPath, _ = helpers.MakeTempTestDir()
conf.configPath, _ = ioutil.TempDir("/tmp", "minio-test-")
defer os.RemoveAll(conf.configPath)
conf.configFile = path.Join(conf.configPath, "config.json")
if _, err := os.Stat(conf.configFile); os.IsNotExist(err) {

3
pkg/utils/cpu/cpu.go
View File

@ -21,14 +21,17 @@ package cpu
// int has_avx2 (void);
import "C"
// CPUID instruction verification wrapper for SSE41 extensions
func HasSSE41() bool {
return int(C.has_sse41()) == 1
}
// CPUID instruction verification wrapper for AVX extensions
func HasAVX() bool {
return int(C.has_avx()) == 1
}
// CPUID instruction verification wrapper for AVX2 extensions
func HasAVX2() bool {
return int(C.has_avx2()) == 1
}

1
pkg/utils/crypto/md5/md5.go
View File

@ -21,6 +21,7 @@ import (
"io"
)
// Low memory footprint io.Reader based md5sum helper
func Sum(reader io.Reader) ([]byte, error) {
hash := md5.New()
var err error

10
pkg/utils/crypto/sha1/sha1.go
View File

@ -40,6 +40,7 @@ type digest struct {
len uint64
}
// Reset digest
func (d *digest) Reset() {
d.h[0] = init0
d.h[1] = init1
@ -68,10 +69,13 @@ func block(dig *digest, p []byte) {
}
}
// Return output size
func (d *digest) Size() int { return Size }
// Return checksum blocksize
func (d *digest) BlockSize() int { return BlockSize }
// Write to digest
func (d *digest) Write(p []byte) (nn int, err error) {
nn = len(p)
d.len += uint64(nn)
@ -95,6 +99,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
return
}
// Return checksum bytes
func (d0 *digest) Sum(in []byte) []byte {
// Make a copy of d0 so that caller can keep writing and summing.
d := *d0
@ -102,6 +107,7 @@ func (d0 *digest) Sum(in []byte) []byte {
return append(in, hash[:]...)
}
// Intermediate checksum function
func (d *digest) checkSum() [Size]byte {
len := d.len
// Padding. Add a 1 bit and 0 bits until 56 bytes mod 64.
@ -135,8 +141,9 @@ func (d *digest) checkSum() [Size]byte {
return digest
}
// Convenience functions
/// Convenience functions
// Single caller sha1 helper
func Sum1(data []byte) [Size]byte {
var d digest
d.Reset()
@ -144,6 +151,7 @@ func Sum1(data []byte) [Size]byte {
return d.checkSum()
}
// io.Reader based streaming sha1 helper
func Sum(reader io.Reader) ([]byte, error) {
h := New()
var err error

65
pkg/utils/crypto/sha1/sha1_gen.go
View File

@ -1,65 +0,0 @@
// +build ignore
//
// Mini Object Storage, (C) 2015 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"os"
"time"
sha1intel "github.com/minio-io/minio/pkg/utils/crypto/sha1"
)
func Sum(reader io.Reader) ([]byte, error) {
k := sha1.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
k.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return k.Sum(nil), nil
}
func main() {
fmt.Println("-- start")
file1, _ := os.Open("filename1")
defer file1.Close()
stark := time.Now()
sum, _ := Sum(file1)
endk := time.Since(stark)
file2, _ := os.Open("filename2")
defer file2.Close()
starth := time.Now()
sumAVX2, _ := sha1intel.Sum(file2)
endh := time.Since(starth)
fmt.Println("std(", endk, ")", "avx2(", endh, ")")
fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumAVX2))
}

14
pkg/utils/crypto/sha256/sha256.go
View File

@ -43,6 +43,7 @@ type digest struct {
len uint64
}
// Reset digest back to default
func (d *digest) Reset() {
d.h[0] = init0
d.h[1] = init1
@ -74,12 +75,13 @@ func New() hash.Hash {
return d
}
func (d *digest) Size() int {
return Size
}
// Return size of checksum
func (d *digest) Size() int { return Size }
// Return blocksize of checksum
func (d *digest) BlockSize() int { return BlockSize }
// Write to digest
func (d *digest) Write(p []byte) (nn int, err error) {
nn = len(p)
d.len += uint64(nn)
@ -103,6 +105,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
return
}
// Return sha256 sum in bytes
func (d0 *digest) Sum(in []byte) []byte {
// Make a copy of d0 so that caller can keep writing and summing.
d := *d0
@ -110,6 +113,7 @@ func (d0 *digest) Sum(in []byte) []byte {
return append(in, hash[:]...)
}
// Intermediate checksum function
func (d *digest) checkSum() [Size]byte {
len := d.len
// Padding. Add a 1 bit and 0 bits until 56 bytes mod 64.
@ -145,8 +149,9 @@ func (d *digest) checkSum() [Size]byte {
return digest
}
// Convenience functions
/// Convenience functions
// Single caller sha256 helper
func Sum256(data []byte) [Size]byte {
var d digest
d.Reset()
@ -154,6 +159,7 @@ func Sum256(data []byte) [Size]byte {
return d.checkSum()
}
// io.Reader based streaming sha256 helper
func Sum(reader io.Reader) ([]byte, error) {
h := New()
var err error

81
pkg/utils/crypto/sha256/sha256_gen.go
View File

@ -1,81 +0,0 @@
// +build ignore
//
// Mini Object Storage, (C) 2015 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"time"
sha256intel "github.com/minio-io/minio/pkg/utils/crypto/sha256"
)
func SumIntel(reader io.Reader) ([]byte, error) {
h := sha256intel.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}
func Sum(reader io.Reader) ([]byte, error) {
k := sha256.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
k.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return k.Sum(nil), nil
}
func main() {
fmt.Println("-- start")
file1, _ := os.Open("filename1")
defer file1.Close()
stark := time.Now()
sum, _ := Sum(file1)
endk := time.Since(stark)
file2, _ := os.Open("filename2")
defer file2.Close()
starth := time.Now()
sumSSE, _ := SumIntel(file2)
endh := time.Since(starth)
fmt.Println("std(", endk, ")", "ssse3(", endh, ")")
fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE))
}

15
pkg/utils/crypto/sha512/sha512.go
View File

@ -55,6 +55,7 @@ func block(dig *digest, p []byte) {
}
}
// Reset digest to its default value
func (d *digest) Reset() {
d.h[0] = init0
d.h[1] = init1
@ -75,12 +76,13 @@ func New() hash.Hash {
return d
}
func (d *digest) Size() int {
return Size
}
// Return output array byte size
func (d *digest) Size() int { return Size }
// Return blockSize
func (d *digest) BlockSize() int { return BlockSize }
// Write blocks
func (d *digest) Write(p []byte) (nn int, err error) {
nn = len(p)
d.len += uint64(nn)
@ -104,6 +106,7 @@ func (d *digest) Write(p []byte) (nn int, err error) {
return
}
// Calculate sha512
func (d0 *digest) Sum(in []byte) []byte {
// Make a copy of d0 so that caller can keep writing and summing.
d := new(digest)
@ -112,6 +115,7 @@ func (d0 *digest) Sum(in []byte) []byte {
return append(in, hash[:]...)
}
// internal checksum calculation, returns [Size]byte
func (d *digest) checkSum() [Size]byte {
// Padding. Add a 1 bit and 0 bits until 112 bytes mod 128.
len := d.len
@ -151,8 +155,9 @@ func (d *digest) checkSum() [Size]byte {
return digest
}
// Convenience functions
/// Convenience functions
// Single caller sha512 helper
func Sum512(data []byte) [Size]byte {
var d digest
d.Reset()
@ -160,6 +165,7 @@ func Sum512(data []byte) [Size]byte {
return d.checkSum()
}
// io.Reader based streaming sha512 helper
func Sum(reader io.Reader) ([]byte, error) {
h := New()
var err error
@ -176,6 +182,7 @@ func Sum(reader io.Reader) ([]byte, error) {
return h.Sum(nil), nil
}
// Similar to 'Sum()' but returns a [Size]byte
func SumStream(reader io.Reader) ([Size]byte, error) {
var returnValue [Size]byte
sumSlice, err := Sum(reader)

81
pkg/utils/crypto/sha512/sha512_gen.go
View File

@ -1,81 +0,0 @@
// +build ignore
//
// Mini Object Storage, (C) 2015 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"os"
"time"
sha512intel "github.com/minio-io/minio/pkg/utils/crypto/sha512"
)
func SumIntel(reader io.Reader) ([]byte, error) {
h := sha512intel.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
h.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return h.Sum(nil), nil
}
func Sum(reader io.Reader) ([]byte, error) {
k := sha512.New()
var err error
for err == nil {
length := 0
byteBuffer := make([]byte, 1024*1024)
length, err = reader.Read(byteBuffer)
byteBuffer = byteBuffer[0:length]
k.Write(byteBuffer)
}
if err != io.EOF {
return nil, err
}
return k.Sum(nil), nil
}
func main() {
fmt.Println("-- start")
file1, _ := os.Open("filename1")
defer file1.Close()
stark := time.Now()
sum, _ := Sum(file1)
endk := time.Since(stark)
file2, _ := os.Open("filename2")
defer file2.Close()
starth := time.Now()
sumSSE, _ := SumIntel(file2)
endh := time.Since(starth)
fmt.Println("std(", endk, ")", "ssse3(", endh, ")")
fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE))
}

1
pkg/utils/crypto/x509/generator.go
View File

@ -72,6 +72,7 @@ func pemBlockForKey(priv interface{}) *pem.Block {
}
}
// Generate certificates using custom parameters
func (tls *Certificates) GenerateCertificates(params X509Params) error {
var rsaBits int = 2048
var priv interface{}

50
pkg/utils/helpers/common.go
View File

@ -1,50 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package helpers
import (
"io/ioutil"
"strings"
)
// Create a new temp directory
func MakeTempTestDir() (string, error) {
return ioutil.TempDir("/tmp", "minio-test-")
}
// Camelcase input string
func FirstUpper(str string) string {
return strings.ToUpper(str[0:1]) + str[1:]
}
func AppendUniqInt(slice []int, i int) []int {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}
func AppendUniqStr(slice []string, i string) []string {
for _, ele := range slice {
if ele == i {
return slice
}
}
return append(slice, i)
}

1
pkg/utils/policy/date.go
View File

@ -13,6 +13,7 @@ type Date struct {
Day byte
}
// Date to string output in yyyy-mm-dd format
func (d Date) String() string {
return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}