mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
Merge pull request #1119 from harshavardhana/minio
xl: Moved to minio/minio
This commit is contained in:
Harshavardhana
commit
f98675660b
2
.gitignore
vendored
2
.gitignore
vendored
@ -11,3 +11,5 @@ site/
|
||||
**/access.log
|
||||
ui-assets.go
|
||||
ui-assets.asc
|
||||
build
|
||||
isa-l
|
||||
14
.travis.yml
14
.travis.yml
@ -1,10 +1,21 @@
|
||||
sudo: false
|
||||
sudo: required
|
||||
dist: trusty
|
||||
language: go
|
||||
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
|
||||
before_install:
|
||||
- git clone https://github.com/yasm/yasm
|
||||
- cd yasm
|
||||
- git checkout v1.3.0
|
||||
- "./autogen.sh"
|
||||
- "./configure"
|
||||
- make
|
||||
- export PATH=$PATH:`pwd`
|
||||
- cd ..
|
||||
|
||||
osx_image: xcode7.2
|
||||
|
||||
env:
|
||||
@ -16,7 +27,6 @@ script:
|
||||
- make test GOFLAGS="-race"
|
||||
|
||||
go:
|
||||
- 1.5.2
|
||||
- 1.5.3
|
||||
|
||||
notifications:
|
||||
|
||||
30
Makefile
30
Makefile
@ -48,6 +48,15 @@ fmt:
|
||||
@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
|
||||
@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
|
||||
|
||||
## Configure Intel library.
|
||||
isa-l:
|
||||
@echo "Configuring $@:"
|
||||
@git clone -q https://github.com/minio/isa-l.git
|
||||
@mkdir -p build
|
||||
@cd build; $(PWD)/isa-l/configure --prefix $(PWD)/build/lib --sysconfdir $(PWD)/build/lib --includedir $(PWD)/build/lib --libdir $(PWD)/build/lib >/dev/null
|
||||
@make -C build >/dev/null
|
||||
@make -C build install >/dev/null
|
||||
|
||||
lint:
|
||||
@echo "Running $@:"
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
|
||||
@ -58,8 +67,8 @@ cyclo:
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
|
||||
|
||||
build: getdeps verifiers $(UI_ASSETS)
|
||||
@echo "Installing minio:"
|
||||
build: getdeps verifiers $(UI_ASSETS) isa-l
|
||||
@GO15VENDOREXPERIMENT=1 go generate ./...
|
||||
|
||||
deadcode:
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
|
||||
@ -69,12 +78,13 @@ spelling:
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
|
||||
|
||||
test: build
|
||||
@echo "Running all testing:"
|
||||
@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
|
||||
@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
|
||||
@echo "Running all minio testing:"
|
||||
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
|
||||
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
|
||||
|
||||
gomake-all: build
|
||||
@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
|
||||
@echo "Installing minio:"
|
||||
@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
|
||||
|
||||
pkg-add:
|
||||
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
|
||||
@ -100,9 +110,9 @@ release:
|
||||
|
||||
clean:
|
||||
@echo "Cleaning up all the generated files:"
|
||||
@rm -fv cover.out
|
||||
@rm -fv minio
|
||||
@rm -fv minio.test
|
||||
@rm -fv pkg/fs/fs.test
|
||||
@rm -fv minio minio.test cover.out
|
||||
@find . -name '*.test' | xargs rm -fv
|
||||
@rm -fv ui-assets.go
|
||||
@rm -fv ui-assets.asc
|
||||
@rm -rf isa-l
|
||||
@rm -rf build
|
||||
|
||||
@ -23,7 +23,7 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
type accessLogHandler struct {
|
||||
|
||||
@ -21,7 +21,7 @@ import (
|
||||
"encoding/base64"
|
||||
"regexp"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@ -26,8 +26,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/fs"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
v4 "github.com/minio/minio/pkg/signature"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -125,7 +125,7 @@ func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
|
||||
}
|
||||
|
||||
// initSignatureV4 initializing signature verification.
|
||||
func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
|
||||
func initSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
|
||||
// strip auth from authorization header.
|
||||
authHeaderValue := req.Header.Get("Authorization")
|
||||
|
||||
@ -156,7 +156,7 @@ func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
|
||||
return nil, err.Trace(authHeaderValue)
|
||||
}
|
||||
if config.Credentials.AccessKeyID == accessKeyID {
|
||||
signature := &fs.Signature{
|
||||
signature := &v4.Signature{
|
||||
AccessKeyID: config.Credentials.AccessKeyID,
|
||||
SecretAccessKey: config.Credentials.SecretAccessKey,
|
||||
Region: region,
|
||||
@ -203,7 +203,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
|
||||
if e != nil {
|
||||
return probe.NewError(e)
|
||||
}
|
||||
postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
|
||||
postPolicyForm, err := v4.ParsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
return err.Trace()
|
||||
}
|
||||
@ -244,7 +244,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
|
||||
}
|
||||
|
||||
// initPostPresignedPolicyV4 initializing post policy signature verification
|
||||
func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
|
||||
func initPostPresignedPolicyV4(formValues map[string]string) (*v4.Signature, *probe.Error) {
|
||||
credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
|
||||
if len(credentialElements) != 5 {
|
||||
return nil, probe.NewError(errCredentialTagMalformed)
|
||||
@ -259,7 +259,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
|
||||
}
|
||||
region := credentialElements[2]
|
||||
if config.Credentials.AccessKeyID == accessKeyID {
|
||||
signature := &fs.Signature{
|
||||
signature := &v4.Signature{
|
||||
AccessKeyID: config.Credentials.AccessKeyID,
|
||||
SecretAccessKey: config.Credentials.SecretAccessKey,
|
||||
Region: region,
|
||||
@ -272,7 +272,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
|
||||
}
|
||||
|
||||
// initPresignedSignatureV4 initializing presigned signature verification
|
||||
func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
|
||||
func initPresignedSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
|
||||
credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
|
||||
if len(credentialElements) != 5 {
|
||||
return nil, probe.NewError(errCredentialTagMalformed)
|
||||
@ -289,7 +289,7 @@ func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
|
||||
signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
|
||||
signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
|
||||
if config.Credentials.AccessKeyID == accessKeyID {
|
||||
signature := &fs.Signature{
|
||||
signature := &v4.Signature{
|
||||
AccessKeyID: config.Credentials.AccessKeyID,
|
||||
SecretAccessKey: config.Credentials.SecretAccessKey,
|
||||
Region: region,
|
||||
|
||||
28
appveyor.yml
28
appveyor.yml
@ -1,30 +1,46 @@
|
||||
# version format
|
||||
version: "{build}"
|
||||
|
||||
# Operating system (build VM template)
|
||||
os: Windows Server 2012 R2
|
||||
os: Visual Studio 2015
|
||||
|
||||
platform: x64
|
||||
|
||||
clone_folder: c:\gopath\src\github.com\minio\minio
|
||||
|
||||
# environment variables
|
||||
environment:
|
||||
GOPATH: c:\gopath
|
||||
GO_EXTLINK_ENABLED: 0
|
||||
GO15VENDOREXPERIMENT: 1
|
||||
UI_ASSETS: ui-assets.go
|
||||
UI_ASSETS_ARMOR: ui-assets.asc
|
||||
|
||||
# scripts that run after cloning repository
|
||||
install:
|
||||
- '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
|
||||
- curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
|
||||
- set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
|
||||
- curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
|
||||
- 7z x -oC:\ mingw-w64.7z > NUL
|
||||
- set PATH=C:\mingw64\bin;%PATH%
|
||||
- x86_64-w64-mingw32-gcc --version
|
||||
- go version
|
||||
- go env
|
||||
- git clone https://github.com/minio/isa-l
|
||||
- cd isa-l
|
||||
- make -f Makefile.unx arch=mingw
|
||||
- mv include isa-l
|
||||
- set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
|
||||
- set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
|
||||
- set CC=x86_64-w64-mingw32-gcc
|
||||
- set CXX=x86_64-w64-mingw32-g++
|
||||
- cd %GOPATH%\src\github.com\minio\minio
|
||||
|
||||
# to run your custom scripts instead of automatic MSBuild
|
||||
build_script:
|
||||
- curl -o ui-assets.go -L https://dl.minio.io/assets/server/ui/%UI_ASSETS%
|
||||
- curl -o ui-assets.asc -L https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
|
||||
- curl -fsSL -o ui-assets.go https://dl.minio.io/assets/server/ui/%UI_ASSETS%
|
||||
- curl -fsSL -o ui-assets.asc https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
|
||||
- gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
|
||||
- gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%
|
||||
- go generate ./...
|
||||
- go test .
|
||||
- go test -race .
|
||||
- go test github.com/minio/minio/pkg...
|
||||
|
||||
@ -22,9 +22,10 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio-xl/pkg/crypto/sha256"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/crypto/sha256"
|
||||
"github.com/minio/minio/pkg/fs"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
v4 "github.com/minio/minio/pkg/signature"
|
||||
)
|
||||
|
||||
// GetBucketLocationHandler - GET Bucket location.
|
||||
@ -204,7 +205,7 @@ func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
var signature *fs.Signature
|
||||
var signature *v4.Signature
|
||||
// Init signature V4 verification
|
||||
if isRequestSignatureV4(req) {
|
||||
var err *probe.Error
|
||||
@ -340,7 +341,7 @@ func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *h
|
||||
writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
|
||||
case fs.BadDigest:
|
||||
writeErrorResponse(w, req, BadDigest, req.URL.Path)
|
||||
case fs.SignatureDoesNotMatch:
|
||||
case v4.SigDoesNotMatch:
|
||||
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
|
||||
case fs.IncompleteBody:
|
||||
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
|
||||
|
||||
@ -20,7 +20,7 @@ import (
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// Configure logger
|
||||
|
||||
66
damerau-levenshtein.go
Normal file
66
damerau-levenshtein.go
Normal file
@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Minio Client (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// Returns the minimum value of a slice of integers
|
||||
func minimum(integers []int) (minVal int) {
|
||||
minVal = math.MaxInt32
|
||||
for _, v := range integers {
|
||||
if v < minVal {
|
||||
minVal = v
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
|
||||
// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
|
||||
func DamerauLevenshteinDistance(a string, b string) int {
|
||||
d := make([][]int, len(a)+1)
|
||||
for i := 1; i <= len(a)+1; i++ {
|
||||
d[i-1] = make([]int, len(b)+1)
|
||||
}
|
||||
for i := 0; i <= len(a); i++ {
|
||||
d[i][0] = i
|
||||
}
|
||||
for j := 0; j <= len(b); j++ {
|
||||
d[0][j] = j
|
||||
}
|
||||
for i := 1; i <= len(a); i++ {
|
||||
for j := 1; j <= len(b); j++ {
|
||||
cost := 0
|
||||
if a[i-1] == b[j-1] {
|
||||
cost = 0
|
||||
} else {
|
||||
cost = 1
|
||||
}
|
||||
d[i][j] = minimum([]int{
|
||||
d[i-1][j] + 1,
|
||||
d[i][j-1] + 1,
|
||||
d[i-1][j-1] + cost,
|
||||
})
|
||||
if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
|
||||
d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
|
||||
}
|
||||
}
|
||||
}
|
||||
return d[len(a)][len(b)]
|
||||
}
|
||||
@ -22,8 +22,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/fs"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
2
jwt.go
2
jwt.go
@ -22,7 +22,7 @@ import (
|
||||
"time"
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
type localFile struct {
|
||||
|
||||
@ -20,7 +20,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
)
|
||||
|
||||
@ -23,7 +23,7 @@ import (
|
||||
"log/syslog"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// syslogHook to send logs via syslog.
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
|
||||
package main
|
||||
|
||||
import "github.com/minio/minio-xl/pkg/probe"
|
||||
import "github.com/minio/minio/pkg/probe"
|
||||
|
||||
func log2Syslog(network, raddr string) *probe.Error {
|
||||
return probe.NewError(errSysLogNotSupported)
|
||||
|
||||
@ -21,7 +21,7 @@ import (
|
||||
"reflect"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
type fields map[string]interface{}
|
||||
|
||||
@ -22,7 +22,7 @@ import (
|
||||
"errors"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
16
main.go
16
main.go
@ -20,11 +20,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// Help template for minio.
|
||||
@ -101,6 +102,19 @@ func findClosestCommands(command string) []string {
|
||||
for _, value := range commandsTree.PrefixMatch(command) {
|
||||
closestCommands = append(closestCommands, value.(string))
|
||||
}
|
||||
sort.Strings(closestCommands)
|
||||
// Suggest other close commands - allow missed, wrongly added and
|
||||
// even transposed characters
|
||||
for _, value := range commandsTree.walk(commandsTree.root) {
|
||||
if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
|
||||
continue
|
||||
}
|
||||
// 2 is arbitrary and represents the max
|
||||
// allowed number of typed errors
|
||||
if DamerauLevenshteinDistance(command, value.(string)) < 2 {
|
||||
closestCommands = append(closestCommands, value.(string))
|
||||
}
|
||||
}
|
||||
return closestCommands
|
||||
}
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
"github.com/olekukonko/ts"
|
||||
)
|
||||
|
||||
|
||||
@ -22,8 +22,9 @@ import (
|
||||
"strconv"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/fs"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
v4 "github.com/minio/minio/pkg/signature"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -172,7 +173,7 @@ func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
var signature *fs.Signature
|
||||
var signature *v4.Signature
|
||||
if isRequestSignatureV4(req) {
|
||||
// Init signature V4 verification
|
||||
var err *probe.Error
|
||||
@ -209,7 +210,7 @@ func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Req
|
||||
writeErrorResponse(w, req, BadDigest, req.URL.Path)
|
||||
case fs.MissingDateHeader:
|
||||
writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
|
||||
case fs.SignatureDoesNotMatch:
|
||||
case v4.SigDoesNotMatch:
|
||||
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
|
||||
case fs.IncompleteBody:
|
||||
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
|
||||
@ -320,7 +321,7 @@ func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http
|
||||
}
|
||||
}
|
||||
|
||||
var signature *fs.Signature
|
||||
var signature *v4.Signature
|
||||
if isRequestSignatureV4(req) {
|
||||
// Init signature V4 verification
|
||||
var err *probe.Error
|
||||
@ -353,7 +354,7 @@ func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http
|
||||
writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
|
||||
case fs.BadDigest:
|
||||
writeErrorResponse(w, req, BadDigest, req.URL.Path)
|
||||
case fs.SignatureDoesNotMatch:
|
||||
case v4.SigDoesNotMatch:
|
||||
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
|
||||
case fs.IncompleteBody:
|
||||
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
|
||||
@ -475,7 +476,7 @@ func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
objectResourcesMetadata := getObjectResources(req.URL.Query())
|
||||
var signature *fs.Signature
|
||||
var signature *v4.Signature
|
||||
if isRequestSignatureV4(req) {
|
||||
// Init signature V4 verification
|
||||
var err *probe.Error
|
||||
@ -516,7 +517,7 @@ func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter,
|
||||
writeErrorResponse(w, req, InvalidPart, req.URL.Path)
|
||||
case fs.InvalidPartOrder:
|
||||
writeErrorResponse(w, req, InvalidPartOrder, req.URL.Path)
|
||||
case fs.SignatureDoesNotMatch:
|
||||
case v4.SigDoesNotMatch:
|
||||
writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
|
||||
case fs.IncompleteBody:
|
||||
writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Client (C) 2015 Minio, Inc.
|
||||
* Minio Cloud Storage (C) 2015-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -34,7 +34,7 @@ type MySuite struct {
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) SetUpSuite(c *C) {
|
||||
root, err := ioutil.TempDir("/tmp", "atomic-")
|
||||
root, err := ioutil.TempDir(os.TempDir(), "atomic-")
|
||||
c.Assert(err, IsNil)
|
||||
s.root = root
|
||||
}
|
||||
@ -23,7 +23,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/cpu"
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
1
pkg/crypto/sha1/.gitignore
vendored
Normal file
1
pkg/crypto/sha1/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.syso
|
||||
150
pkg/crypto/sha1/sha1.go
Normal file
150
pkg/crypto/sha1/sha1.go
Normal file
@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file of
|
||||
// Golang project:
|
||||
// https://github.com/golang/go/blob/master/LICENSE
|
||||
|
||||
// Using this part of Minio codebase under the license
|
||||
// Apache License Version 2.0 with modifications
|
||||
|
||||
// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
|
||||
package sha1
|
||||
|
||||
import "hash"
|
||||
|
||||
// Size - The size of a SHA1 checksum in bytes.
|
||||
const Size = 20
|
||||
|
||||
// BlockSize - The blocksize of SHA1 in bytes.
|
||||
const BlockSize = 64
|
||||
|
||||
const (
|
||||
chunk = 64
|
||||
init0 = 0x67452301
|
||||
init1 = 0xEFCDAB89
|
||||
init2 = 0x98BADCFE
|
||||
init3 = 0x10325476
|
||||
init4 = 0xC3D2E1F0
|
||||
)
|
||||
|
||||
// digest represents the partial evaluation of a checksum.
|
||||
type digest struct {
|
||||
h [5]uint32
|
||||
x [chunk]byte
|
||||
nx int
|
||||
len uint64
|
||||
}
|
||||
|
||||
// Reset digest
|
||||
func (d *digest) Reset() {
|
||||
d.h[0] = init0
|
||||
d.h[1] = init1
|
||||
d.h[2] = init2
|
||||
d.h[3] = init3
|
||||
d.h[4] = init4
|
||||
d.nx = 0
|
||||
d.len = 0
|
||||
}
|
||||
|
||||
// New returns a new hash.Hash computing the SHA1 checksum.
|
||||
func New() hash.Hash {
|
||||
d := new(digest)
|
||||
d.Reset()
|
||||
return d
|
||||
}
|
||||
|
||||
// Return output size
|
||||
func (d *digest) Size() int { return Size }
|
||||
|
||||
// Return checksum blocksize
|
||||
func (d *digest) BlockSize() int { return BlockSize }
|
||||
|
||||
// Write to digest
|
||||
func (d *digest) Write(p []byte) (nn int, err error) {
|
||||
nn = len(p)
|
||||
d.len += uint64(nn)
|
||||
if d.nx > 0 {
|
||||
n := copy(d.x[d.nx:], p)
|
||||
d.nx += n
|
||||
if d.nx == chunk {
|
||||
block(d, d.x[:])
|
||||
d.nx = 0
|
||||
}
|
||||
p = p[n:]
|
||||
}
|
||||
if len(p) >= chunk {
|
||||
n := len(p) &^ (chunk - 1)
|
||||
block(d, p[:n])
|
||||
p = p[n:]
|
||||
}
|
||||
if len(p) > 0 {
|
||||
d.nx = copy(d.x[:], p)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Return checksum bytes
|
||||
func (d *digest) Sum(in []byte) []byte {
|
||||
// Make a copy of d0 so that caller can keep writing and summing.
|
||||
d0 := *d
|
||||
hash := d0.checkSum()
|
||||
return append(in, hash[:]...)
|
||||
}
|
||||
|
||||
// Intermediate checksum function
|
||||
func (d *digest) checkSum() [Size]byte {
|
||||
len := d.len
|
||||
// Padding. Add a 1 bit and 0 bits until 56 bytes mod 64.
|
||||
var tmp [64]byte
|
||||
tmp[0] = 0x80
|
||||
if len%64 < 56 {
|
||||
d.Write(tmp[0 : 56-len%64])
|
||||
} else {
|
||||
d.Write(tmp[0 : 64+56-len%64])
|
||||
}
|
||||
|
||||
// Length in bits.
|
||||
len <<= 3
|
||||
for i := uint(0); i < 8; i++ {
|
||||
tmp[i] = byte(len >> (56 - 8*i))
|
||||
}
|
||||
d.Write(tmp[0:8])
|
||||
|
||||
if d.nx != 0 {
|
||||
panic("d.nx != 0")
|
||||
}
|
||||
|
||||
var digest [Size]byte
|
||||
for i, s := range d.h {
|
||||
digest[i*4] = byte(s >> 24)
|
||||
digest[i*4+1] = byte(s >> 16)
|
||||
digest[i*4+2] = byte(s >> 8)
|
||||
digest[i*4+3] = byte(s)
|
||||
}
|
||||
|
||||
return digest
|
||||
}
|
||||
|
||||
// Sum - single caller sha1 helper
|
||||
func Sum(data []byte) [Size]byte {
|
||||
var d digest
|
||||
d.Reset()
|
||||
d.Write(data)
|
||||
return d.checkSum()
|
||||
}
|
||||
967
pkg/crypto/sha1/sha1_linux.S
Normal file
967
pkg/crypto/sha1/sha1_linux.S
Normal file
@ -0,0 +1,967 @@
|
||||
/*
|
||||
* Implement fast SHA-1 with AVX2 instructions. (x86_64)
|
||||
*
|
||||
* This file is provided under a dual BSD/GPLv2 license. When using or
|
||||
* redistributing this file, you may do so under either license.
|
||||
*
|
||||
* GPL LICENSE SUMMARY
|
||||
*
|
||||
* Copyright(c) 2014 Intel Corporation.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of version 2 of the GNU General Public License as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*
|
||||
* Contact Information:
|
||||
* Ilya Albrekht <ilya.albrekht@intel.com>
|
||||
* Maxim Locktyukhin <maxim.locktyukhin@intel.com>
|
||||
* Ronen Zohar <ronen.zohar@intel.com>
|
||||
* Chandramouli Narayanan <mouli@linux.intel.com>
|
||||
*
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) 2014 Intel Corporation.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
|
||||
*
|
||||
*This implementation is based on the previous SSSE3 release:
|
||||
* https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
|
||||
*
|
||||
*Updates 20-byte SHA-1 record in 'hash' for even number of
|
||||
*'num_blocks' consecutive 64-byte blocks
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Using this part of Minio codebase under the license
|
||||
* Apache License Version 2.0 with modifications
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef HAS_AVX2
|
||||
#ifndef ENTRY
|
||||
#define ENTRY(name) \
|
||||
.globl name ; \
|
||||
.align 4,0x90 ; \
|
||||
name:
|
||||
#endif
|
||||
|
||||
#ifndef END
|
||||
#define END(name) \
|
||||
.size name, .-name
|
||||
#endif
|
||||
|
||||
#ifndef ENDPROC
|
||||
#define ENDPROC(name) \
|
||||
.type name, @function ; \
|
||||
END(name)
|
||||
#endif
|
||||
|
||||
#define NUM_INVALID 100
|
||||
|
||||
#define TYPE_R32 0
|
||||
#define TYPE_R64 1
|
||||
#define TYPE_XMM 2
|
||||
#define TYPE_INVALID 100
|
||||
|
||||
.macro R32_NUM opd r32
|
||||
\opd = NUM_INVALID
|
||||
.ifc \r32,%eax
|
||||
\opd = 0
|
||||
.endif
|
||||
.ifc \r32,%ecx
|
||||
\opd = 1
|
||||
.endif
|
||||
.ifc \r32,%edx
|
||||
\opd = 2
|
||||
.endif
|
||||
.ifc \r32,%ebx
|
||||
\opd = 3
|
||||
.endif
|
||||
.ifc \r32,%esp
|
||||
\opd = 4
|
||||
.endif
|
||||
.ifc \r32,%ebp
|
||||
\opd = 5
|
||||
.endif
|
||||
.ifc \r32,%esi
|
||||
\opd = 6
|
||||
.endif
|
||||
.ifc \r32,%edi
|
||||
\opd = 7
|
||||
.endif
|
||||
#ifdef X86_64
|
||||
.ifc \r32,%r8d
|
||||
\opd = 8
|
||||
.endif
|
||||
.ifc \r32,%r9d
|
||||
\opd = 9
|
||||
.endif
|
||||
.ifc \r32,%r10d
|
||||
\opd = 10
|
||||
.endif
|
||||
.ifc \r32,%r11d
|
||||
\opd = 11
|
||||
.endif
|
||||
.ifc \r32,%r12d
|
||||
\opd = 12
|
||||
.endif
|
||||
.ifc \r32,%r13d
|
||||
\opd = 13
|
||||
.endif
|
||||
.ifc \r32,%r14d
|
||||
\opd = 14
|
||||
.endif
|
||||
.ifc \r32,%r15d
|
||||
\opd = 15
|
||||
.endif
|
||||
#endif
|
||||
.endm
|
||||
|
||||
.macro R64_NUM opd r64
|
||||
\opd = NUM_INVALID
|
||||
#ifdef X86_64
|
||||
.ifc \r64,%rax
|
||||
\opd = 0
|
||||
.endif
|
||||
.ifc \r64,%rcx
|
||||
\opd = 1
|
||||
.endif
|
||||
.ifc \r64,%rdx
|
||||
\opd = 2
|
||||
.endif
|
||||
.ifc \r64,%rbx
|
||||
\opd = 3
|
||||
.endif
|
||||
.ifc \r64,%rsp
|
||||
\opd = 4
|
||||
.endif
|
||||
.ifc \r64,%rbp
|
||||
\opd = 5
|
||||
.endif
|
||||
.ifc \r64,%rsi
|
||||
\opd = 6
|
||||
.endif
|
||||
.ifc \r64,%rdi
|
||||
\opd = 7
|
||||
.endif
|
||||
.ifc \r64,%r8
|
||||
\opd = 8
|
||||
.endif
|
||||
.ifc \r64,%r9
|
||||
\opd = 9
|
||||
.endif
|
||||
.ifc \r64,%r10
|
||||
\opd = 10
|
||||
.endif
|
||||
.ifc \r64,%r11
|
||||
\opd = 11
|
||||
.endif
|
||||
.ifc \r64,%r12
|
||||
\opd = 12
|
||||
.endif
|
||||
.ifc \r64,%r13
|
||||
\opd = 13
|
||||
.endif
|
||||
.ifc \r64,%r14
|
||||
\opd = 14
|
||||
.endif
|
||||
.ifc \r64,%r15
|
||||
\opd = 15
|
||||
.endif
|
||||
#endif
|
||||
.endm
|
||||
|
||||
.macro XMM_NUM opd xmm
|
||||
\opd = NUM_INVALID
|
||||
.ifc \xmm,%xmm0
|
||||
\opd = 0
|
||||
.endif
|
||||
.ifc \xmm,%xmm1
|
||||
\opd = 1
|
||||
.endif
|
||||
.ifc \xmm,%xmm2
|
||||
\opd = 2
|
||||
.endif
|
||||
.ifc \xmm,%xmm3
|
||||
\opd = 3
|
||||
.endif
|
||||
.ifc \xmm,%xmm4
|
||||
\opd = 4
|
||||
.endif
|
||||
.ifc \xmm,%xmm5
|
||||
\opd = 5
|
||||
.endif
|
||||
.ifc \xmm,%xmm6
|
||||
\opd = 6
|
||||
.endif
|
||||
.ifc \xmm,%xmm7
|
||||
\opd = 7
|
||||
.endif
|
||||
.ifc \xmm,%xmm8
|
||||
\opd = 8
|
||||
.endif
|
||||
.ifc \xmm,%xmm9
|
||||
\opd = 9
|
||||
.endif
|
||||
.ifc \xmm,%xmm10
|
||||
\opd = 10
|
||||
.endif
|
||||
.ifc \xmm,%xmm11
|
||||
\opd = 11
|
||||
.endif
|
||||
.ifc \xmm,%xmm12
|
||||
\opd = 12
|
||||
.endif
|
||||
.ifc \xmm,%xmm13
|
||||
\opd = 13
|
||||
.endif
|
||||
.ifc \xmm,%xmm14
|
||||
\opd = 14
|
||||
.endif
|
||||
.ifc \xmm,%xmm15
|
||||
\opd = 15
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro TYPE type reg
|
||||
R32_NUM reg_type_r32 \reg
|
||||
R64_NUM reg_type_r64 \reg
|
||||
XMM_NUM reg_type_xmm \reg
|
||||
.if reg_type_r64 <> NUM_INVALID
|
||||
\type = TYPE_R64
|
||||
.elseif reg_type_r32 <> NUM_INVALID
|
||||
\type = TYPE_R32
|
||||
.elseif reg_type_xmm <> NUM_INVALID
|
||||
\type = TYPE_XMM
|
||||
.else
|
||||
\type = TYPE_INVALID
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro PFX_OPD_SIZE
|
||||
.byte 0x66
|
||||
.endm
|
||||
|
||||
.macro PFX_REX opd1 opd2 W=0
|
||||
.if ((\opd1 | \opd2) & 8) || \W
|
||||
.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro MODRM mod opd1 opd2
|
||||
.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
|
||||
.endm
|
||||
|
||||
.macro PSHUFB_XMM xmm1 xmm2
|
||||
XMM_NUM pshufb_opd1 \xmm1
|
||||
XMM_NUM pshufb_opd2 \xmm2
|
||||
PFX_OPD_SIZE
|
||||
PFX_REX pshufb_opd1 pshufb_opd2
|
||||
.byte 0x0f, 0x38, 0x00
|
||||
MODRM 0xc0 pshufb_opd1 pshufb_opd2
|
||||
.endm
|
||||
|
||||
.macro PCLMULQDQ imm8 xmm1 xmm2
|
||||
XMM_NUM clmul_opd1 \xmm1
|
||||
XMM_NUM clmul_opd2 \xmm2
|
||||
PFX_OPD_SIZE
|
||||
PFX_REX clmul_opd1 clmul_opd2
|
||||
.byte 0x0f, 0x3a, 0x44
|
||||
MODRM 0xc0 clmul_opd1 clmul_opd2
|
||||
.byte \imm8
|
||||
.endm
|
||||
|
||||
.macro PEXTRD imm8 xmm gpr
|
||||
R32_NUM extrd_opd1 \gpr
|
||||
XMM_NUM extrd_opd2 \xmm
|
||||
PFX_OPD_SIZE
|
||||
PFX_REX extrd_opd1 extrd_opd2
|
||||
.byte 0x0f, 0x3a, 0x16
|
||||
MODRM 0xc0 extrd_opd1 extrd_opd2
|
||||
.byte \imm8
|
||||
.endm
|
||||
|
||||
.macro MOVQ_R64_XMM opd1 opd2
|
||||
TYPE movq_r64_xmm_opd1_type \opd1
|
||||
.if movq_r64_xmm_opd1_type == TYPE_XMM
|
||||
XMM_NUM movq_r64_xmm_opd1 \opd1
|
||||
R64_NUM movq_r64_xmm_opd2 \opd2
|
||||
.else
|
||||
R64_NUM movq_r64_xmm_opd1 \opd1
|
||||
XMM_NUM movq_r64_xmm_opd2 \opd2
|
||||
.endif
|
||||
PFX_OPD_SIZE
|
||||
PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
|
||||
.if movq_r64_xmm_opd1_type == TYPE_XMM
|
||||
.byte 0x0f, 0x7e
|
||||
.else
|
||||
.byte 0x0f, 0x6e
|
||||
.endif
|
||||
MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
|
||||
.endm
|
||||
|
||||
#define CTX %rdi /* arg1 */
|
||||
#define BUF %rsi /* arg2 */
|
||||
#define CNT %rdx /* arg3 */
|
||||
|
||||
#define REG_A %ecx
|
||||
#define REG_B %esi
|
||||
#define REG_C %edi
|
||||
#define REG_D %eax
|
||||
#define REG_E %edx
|
||||
#define REG_TB %ebx
|
||||
#define REG_TA %r12d
|
||||
#define REG_RA %rcx
|
||||
#define REG_RB %rsi
|
||||
#define REG_RC %rdi
|
||||
#define REG_RD %rax
|
||||
#define REG_RE %rdx
|
||||
#define REG_RTA %r12
|
||||
#define REG_RTB %rbx
|
||||
#define REG_T1 %ebp
|
||||
#define xmm_mov vmovups
|
||||
#define avx2_zeroupper vzeroupper
|
||||
#define RND_F1 1
|
||||
#define RND_F2 2
|
||||
#define RND_F3 3
|
||||
|
||||
.macro REGALLOC
|
||||
.set A, REG_A
|
||||
.set B, REG_B
|
||||
.set C, REG_C
|
||||
.set D, REG_D
|
||||
.set E, REG_E
|
||||
.set TB, REG_TB
|
||||
.set TA, REG_TA
|
||||
|
||||
.set RA, REG_RA
|
||||
.set RB, REG_RB
|
||||
.set RC, REG_RC
|
||||
.set RD, REG_RD
|
||||
.set RE, REG_RE
|
||||
|
||||
.set RTA, REG_RTA
|
||||
.set RTB, REG_RTB
|
||||
|
||||
.set T1, REG_T1
|
||||
.endm
|
||||
|
||||
#define K_BASE %r8
|
||||
#define HASH_PTR %r9
|
||||
#define BUFFER_PTR %r10
|
||||
#define BUFFER_PTR2 %r13
|
||||
#define BUFFER_END %r11
|
||||
|
||||
#define PRECALC_BUF %r14
|
||||
#define WK_BUF %r15
|
||||
|
||||
#define W_TMP %xmm0
|
||||
#define WY_TMP %ymm0
|
||||
#define WY_TMP2 %ymm9
|
||||
|
||||
# AVX2 variables
|
||||
#define WY0 %ymm3
|
||||
#define WY4 %ymm5
|
||||
#define WY08 %ymm7
|
||||
#define WY12 %ymm8
|
||||
#define WY16 %ymm12
|
||||
#define WY20 %ymm13
|
||||
#define WY24 %ymm14
|
||||
#define WY28 %ymm15
|
||||
|
||||
#define YMM_SHUFB_BSWAP %ymm10
|
||||
|
||||
/*
|
||||
* Keep 2 iterations precalculated at a time:
|
||||
* - 80 DWORDs per iteration * 2
|
||||
*/
|
||||
#define W_SIZE (80*2*2 +16)
|
||||
|
||||
#define WK(t) ((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
|
||||
#define PRECALC_WK(t) ((t)*2*2)(PRECALC_BUF)
|
||||
|
||||
|
||||
.macro UPDATE_HASH hash, val
|
||||
add \hash, \val
|
||||
mov \val, \hash
|
||||
.endm
|
||||
|
||||
.macro PRECALC_RESET_WY
|
||||
.set WY_00, WY0
|
||||
.set WY_04, WY4
|
||||
.set WY_08, WY08
|
||||
.set WY_12, WY12
|
||||
.set WY_16, WY16
|
||||
.set WY_20, WY20
|
||||
.set WY_24, WY24
|
||||
.set WY_28, WY28
|
||||
.set WY_32, WY_00
|
||||
.endm
|
||||
|
||||
.macro PRECALC_ROTATE_WY
|
||||
/* Rotate macros */
|
||||
.set WY_32, WY_28
|
||||
.set WY_28, WY_24
|
||||
.set WY_24, WY_20
|
||||
.set WY_20, WY_16
|
||||
.set WY_16, WY_12
|
||||
.set WY_12, WY_08
|
||||
.set WY_08, WY_04
|
||||
.set WY_04, WY_00
|
||||
.set WY_00, WY_32
|
||||
|
||||
/* Define register aliases */
|
||||
.set WY, WY_00
|
||||
.set WY_minus_04, WY_04
|
||||
.set WY_minus_08, WY_08
|
||||
.set WY_minus_12, WY_12
|
||||
.set WY_minus_16, WY_16
|
||||
.set WY_minus_20, WY_20
|
||||
.set WY_minus_24, WY_24
|
||||
.set WY_minus_28, WY_28
|
||||
.set WY_minus_32, WY
|
||||
.endm
|
||||
|
||||
.macro PRECALC_00_15
|
||||
.if (i == 0) # Initialize and rotate registers
|
||||
PRECALC_RESET_WY
|
||||
PRECALC_ROTATE_WY
|
||||
.endif
|
||||
|
||||
/* message scheduling pre-compute for rounds 0-15 */
|
||||
.if ((i & 7) == 0)
|
||||
/*
|
||||
* blended AVX2 and ALU instruction scheduling
|
||||
* 1 vector iteration per 8 rounds
|
||||
*/
|
||||
vmovdqu ((i * 2) + PRECALC_OFFSET)(BUFFER_PTR), W_TMP
|
||||
.elseif ((i & 7) == 1)
|
||||
vinsertf128 $1, (((i-1) * 2)+PRECALC_OFFSET)(BUFFER_PTR2),\
|
||||
WY_TMP, WY_TMP
|
||||
.elseif ((i & 7) == 2)
|
||||
vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
|
||||
.elseif ((i & 7) == 4)
|
||||
vpaddd K_XMM(K_BASE), WY, WY_TMP
|
||||
.elseif ((i & 7) == 7)
|
||||
vmovdqu WY_TMP, PRECALC_WK(i&~7)
|
||||
|
||||
PRECALC_ROTATE_WY
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro PRECALC_16_31
|
||||
/*
|
||||
* message scheduling pre-compute for rounds 16-31
|
||||
* calculating last 32 w[i] values in 8 XMM registers
|
||||
* pre-calculate K+w[i] values and store to mem
|
||||
* for later load by ALU add instruction
|
||||
*
|
||||
* "brute force" vectorization for rounds 16-31 only
|
||||
* due to w[i]->w[i-3] dependency
|
||||
*/
|
||||
.if ((i & 7) == 0)
|
||||
/*
|
||||
* blended AVX2 and ALU instruction scheduling
|
||||
* 1 vector iteration per 8 rounds
|
||||
*/
|
||||
/* w[i-14] */
|
||||
vpalignr $8, WY_minus_16, WY_minus_12, WY
|
||||
vpsrldq $4, WY_minus_04, WY_TMP /* w[i-3] */
|
||||
.elseif ((i & 7) == 1)
|
||||
vpxor WY_minus_08, WY, WY
|
||||
vpxor WY_minus_16, WY_TMP, WY_TMP
|
||||
.elseif ((i & 7) == 2)
|
||||
vpxor WY_TMP, WY, WY
|
||||
vpslldq $12, WY, WY_TMP2
|
||||
.elseif ((i & 7) == 3)
|
||||
vpslld $1, WY, WY_TMP
|
||||
vpsrld $31, WY, WY
|
||||
.elseif ((i & 7) == 4)
|
||||
vpor WY, WY_TMP, WY_TMP
|
||||
vpslld $2, WY_TMP2, WY
|
||||
.elseif ((i & 7) == 5)
|
||||
vpsrld $30, WY_TMP2, WY_TMP2
|
||||
vpxor WY, WY_TMP, WY_TMP
|
||||
.elseif ((i & 7) == 7)
|
||||
vpxor WY_TMP2, WY_TMP, WY
|
||||
vpaddd K_XMM(K_BASE), WY, WY_TMP
|
||||
vmovdqu WY_TMP, PRECALC_WK(i&~7)
|
||||
|
||||
PRECALC_ROTATE_WY
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro PRECALC_32_79
|
||||
/*
|
||||
* in SHA-1 specification:
|
||||
* w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1
|
||||
* instead we do equal:
|
||||
* w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
|
||||
* allows more efficient vectorization
|
||||
* since w[i]=>w[i-3] dependency is broken
|
||||
*/
|
||||
|
||||
.if ((i & 7) == 0)
|
||||
/*
|
||||
* blended AVX2 and ALU instruction scheduling
|
||||
* 1 vector iteration per 8 rounds
|
||||
*/
|
||||
vpalignr $8, WY_minus_08, WY_minus_04, WY_TMP
|
||||
.elseif ((i & 7) == 1)
|
||||
/* W is W_minus_32 before xor */
|
||||
vpxor WY_minus_28, WY, WY
|
||||
.elseif ((i & 7) == 2)
|
||||
vpxor WY_minus_16, WY_TMP, WY_TMP
|
||||
.elseif ((i & 7) == 3)
|
||||
vpxor WY_TMP, WY, WY
|
||||
.elseif ((i & 7) == 4)
|
||||
vpslld $2, WY, WY_TMP
|
||||
.elseif ((i & 7) == 5)
|
||||
vpsrld $30, WY, WY
|
||||
vpor WY, WY_TMP, WY
|
||||
.elseif ((i & 7) == 7)
|
||||
vpaddd K_XMM(K_BASE), WY, WY_TMP
|
||||
vmovdqu WY_TMP, PRECALC_WK(i&~7)
|
||||
|
||||
PRECALC_ROTATE_WY
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro PRECALC r, s
|
||||
.set i, \r
|
||||
|
||||
.if (i < 40)
|
||||
.set K_XMM, 32*0
|
||||
.elseif (i < 80)
|
||||
.set K_XMM, 32*1
|
||||
.elseif (i < 120)
|
||||
.set K_XMM, 32*2
|
||||
.else
|
||||
.set K_XMM, 32*3
|
||||
.endif
|
||||
|
||||
.if (i<32)
|
||||
PRECALC_00_15 \s
|
||||
.elseif (i<64)
|
||||
PRECALC_16_31 \s
|
||||
.elseif (i < 160)
|
||||
PRECALC_32_79 \s
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro ROTATE_STATE
|
||||
.set T_REG, E
|
||||
.set E, D
|
||||
.set D, C
|
||||
.set C, B
|
||||
.set B, TB
|
||||
.set TB, A
|
||||
.set A, T_REG
|
||||
|
||||
.set T_REG, RE
|
||||
.set RE, RD
|
||||
.set RD, RC
|
||||
.set RC, RB
|
||||
.set RB, RTB
|
||||
.set RTB, RA
|
||||
.set RA, T_REG
|
||||
.endm
|
||||
|
||||
/* Macro relies on saved ROUND_Fx */
|
||||
|
||||
.macro RND_FUN f, r
|
||||
.if (\f == RND_F1)
|
||||
ROUND_F1 \r
|
||||
.elseif (\f == RND_F2)
|
||||
ROUND_F2 \r
|
||||
.elseif (\f == RND_F3)
|
||||
ROUND_F3 \r
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro RR r
|
||||
.set round_id, (\r % 80)
|
||||
|
||||
.if (round_id == 0) /* Precalculate F for first round */
|
||||
.set ROUND_FUNC, RND_F1
|
||||
mov B, TB
|
||||
|
||||
rorx $(32-30), B, B /* b>>>2 */
|
||||
andn D, TB, T1
|
||||
and C, TB
|
||||
xor T1, TB
|
||||
.endif
|
||||
|
||||
RND_FUN ROUND_FUNC, \r
|
||||
ROTATE_STATE
|
||||
|
||||
.if (round_id == 18)
|
||||
.set ROUND_FUNC, RND_F2
|
||||
.elseif (round_id == 38)
|
||||
.set ROUND_FUNC, RND_F3
|
||||
.elseif (round_id == 58)
|
||||
.set ROUND_FUNC, RND_F2
|
||||
.endif
|
||||
|
||||
.set round_id, ( (\r+1) % 80)
|
||||
|
||||
RND_FUN ROUND_FUNC, (\r+1)
|
||||
ROTATE_STATE
|
||||
.endm
|
||||
|
||||
.macro ROUND_F1 r
|
||||
add WK(\r), E
|
||||
|
||||
andn C, A, T1 /* ~b&d */
|
||||
lea (RE,RTB), E /* Add F from the previous round */
|
||||
|
||||
rorx $(32-5), A, TA /* T2 = A >>> 5 */
|
||||
rorx $(32-30),A, TB /* b>>>2 for next round */
|
||||
|
||||
PRECALC (\r) /* msg scheduling for next 2 blocks */
|
||||
|
||||
/*
|
||||
* Calculate F for the next round
|
||||
* (b & c) ^ andn[b, d]
|
||||
*/
|
||||
and B, A /* b&c */
|
||||
xor T1, A /* F1 = (b&c) ^ (~b&d) */
|
||||
|
||||
lea (RE,RTA), E /* E += A >>> 5 */
|
||||
.endm
|
||||
|
||||
.macro ROUND_F2 r
|
||||
add WK(\r), E
|
||||
lea (RE,RTB), E /* Add F from the previous round */
|
||||
|
||||
/* Calculate F for the next round */
|
||||
rorx $(32-5), A, TA /* T2 = A >>> 5 */
|
||||
.if ((round_id) < 79)
|
||||
rorx $(32-30), A, TB /* b>>>2 for next round */
|
||||
.endif
|
||||
PRECALC (\r) /* msg scheduling for next 2 blocks */
|
||||
|
||||
.if ((round_id) < 79)
|
||||
xor B, A
|
||||
.endif
|
||||
|
||||
add TA, E /* E += A >>> 5 */
|
||||
|
||||
.if ((round_id) < 79)
|
||||
xor C, A
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro ROUND_F3 r
|
||||
add WK(\r), E
|
||||
PRECALC (\r) /* msg scheduling for next 2 blocks */
|
||||
|
||||
lea (RE,RTB), E /* Add F from the previous round */
|
||||
|
||||
mov B, T1
|
||||
or A, T1
|
||||
|
||||
rorx $(32-5), A, TA /* T2 = A >>> 5 */
|
||||
rorx $(32-30), A, TB /* b>>>2 for next round */
|
||||
|
||||
/* Calculate F for the next round
|
||||
* (b and c) or (d and (b or c))
|
||||
*/
|
||||
and C, T1
|
||||
and B, A
|
||||
or T1, A
|
||||
|
||||
add TA, E /* E += A >>> 5 */
|
||||
|
||||
.endm
|
||||
|
||||
/*
|
||||
* macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
|
||||
*/
|
||||
.macro SHA1_PIPELINED_MAIN_BODY
|
||||
|
||||
REGALLOC
|
||||
|
||||
mov (HASH_PTR), A
|
||||
mov 4(HASH_PTR), B
|
||||
mov 8(HASH_PTR), C
|
||||
mov 12(HASH_PTR), D
|
||||
mov 16(HASH_PTR), E
|
||||
|
||||
mov %rsp, PRECALC_BUF
|
||||
lea (2*4*80+32)(%rsp), WK_BUF
|
||||
|
||||
# Precalc WK for first 2 blocks
|
||||
PRECALC_OFFSET = 0
|
||||
.set i, 0
|
||||
.rept 160
|
||||
PRECALC i
|
||||
.set i, i + 1
|
||||
.endr
|
||||
PRECALC_OFFSET = 128
|
||||
xchg WK_BUF, PRECALC_BUF
|
||||
|
||||
.align 32
|
||||
_loop:
|
||||
/*
|
||||
* code loops through more than one block
|
||||
* we use K_BASE value as a signal of a last block,
|
||||
* it is set below by: cmovae BUFFER_PTR, K_BASE
|
||||
*/
|
||||
cmp K_BASE, BUFFER_PTR
|
||||
jne _begin
|
||||
.align 32
|
||||
jmp _end
|
||||
.align 32
|
||||
_begin:
|
||||
|
||||
/*
|
||||
* Do first block
|
||||
* rounds: 0,2,4,6,8
|
||||
*/
|
||||
.set j, 0
|
||||
.rept 5
|
||||
RR j
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
jmp _loop0
|
||||
_loop0:
|
||||
|
||||
/*
|
||||
* rounds:
|
||||
* 10,12,14,16,18
|
||||
* 20,22,24,26,28
|
||||
* 30,32,34,36,38
|
||||
* 40,42,44,46,48
|
||||
* 50,52,54,56,58
|
||||
*/
|
||||
.rept 25
|
||||
RR j
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
add $(2*64), BUFFER_PTR /* move to next odd-64-byte block */
|
||||
cmp BUFFER_END, BUFFER_PTR /* is current block the last one? */
|
||||
cmovae K_BASE, BUFFER_PTR /* signal the last iteration smartly */
|
||||
|
||||
/*
|
||||
* rounds
|
||||
* 60,62,64,66,68
|
||||
* 70,72,74,76,78
|
||||
*/
|
||||
.rept 10
|
||||
RR j
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
UPDATE_HASH (HASH_PTR), A
|
||||
UPDATE_HASH 4(HASH_PTR), TB
|
||||
UPDATE_HASH 8(HASH_PTR), C
|
||||
UPDATE_HASH 12(HASH_PTR), D
|
||||
UPDATE_HASH 16(HASH_PTR), E
|
||||
|
||||
cmp K_BASE, BUFFER_PTR /* is current block the last one? */
|
||||
je _loop
|
||||
|
||||
mov TB, B
|
||||
|
||||
/* Process second block */
|
||||
/*
|
||||
* rounds
|
||||
* 0+80, 2+80, 4+80, 6+80, 8+80
|
||||
* 10+80,12+80,14+80,16+80,18+80
|
||||
*/
|
||||
|
||||
.set j, 0
|
||||
.rept 10
|
||||
RR j+80
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
jmp _loop1
|
||||
_loop1:
|
||||
/*
|
||||
* rounds
|
||||
* 20+80,22+80,24+80,26+80,28+80
|
||||
* 30+80,32+80,34+80,36+80,38+80
|
||||
*/
|
||||
.rept 10
|
||||
RR j+80
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
jmp _loop2
|
||||
_loop2:
|
||||
|
||||
/*
|
||||
* rounds
|
||||
* 40+80,42+80,44+80,46+80,48+80
|
||||
* 50+80,52+80,54+80,56+80,58+80
|
||||
*/
|
||||
.rept 10
|
||||
RR j+80
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
add $(2*64), BUFFER_PTR2 /* move to next even-64-byte block */
|
||||
|
||||
cmp BUFFER_END, BUFFER_PTR2 /* is current block the last one */
|
||||
cmovae K_BASE, BUFFER_PTR /* signal the last iteration smartly */
|
||||
|
||||
jmp _loop3
|
||||
_loop3:
|
||||
|
||||
/*
|
||||
* rounds
|
||||
* 60+80,62+80,64+80,66+80,68+80
|
||||
* 70+80,72+80,74+80,76+80,78+80
|
||||
*/
|
||||
.rept 10
|
||||
RR j+80
|
||||
.set j, j+2
|
||||
.endr
|
||||
|
||||
UPDATE_HASH (HASH_PTR), A
|
||||
UPDATE_HASH 4(HASH_PTR), TB
|
||||
UPDATE_HASH 8(HASH_PTR), C
|
||||
UPDATE_HASH 12(HASH_PTR), D
|
||||
UPDATE_HASH 16(HASH_PTR), E
|
||||
|
||||
/* Reset state for AVX2 reg permutation */
|
||||
mov A, TA
|
||||
mov TB, A
|
||||
mov C, TB
|
||||
mov E, C
|
||||
mov D, B
|
||||
mov TA, D
|
||||
|
||||
REGALLOC
|
||||
|
||||
xchg WK_BUF, PRECALC_BUF
|
||||
|
||||
jmp _loop
|
||||
|
||||
.align 32
|
||||
_end:
|
||||
|
||||
.endm
|
||||
|
||||
.section .rodata
|
||||
|
||||
#define K1 0x5a827999
|
||||
#define K2 0x6ed9eba1
|
||||
#define K3 0x8f1bbcdc
|
||||
#define K4 0xca62c1d6
|
||||
|
||||
.align 128
|
||||
K_XMM_AR:
|
||||
.long K1, K1, K1, K1
|
||||
.long K1, K1, K1, K1
|
||||
.long K2, K2, K2, K2
|
||||
.long K2, K2, K2, K2
|
||||
.long K3, K3, K3, K3
|
||||
.long K3, K3, K3, K3
|
||||
.long K4, K4, K4, K4
|
||||
.long K4, K4, K4, K4
|
||||
|
||||
BSWAP_SHUFB_CTL:
|
||||
.long 0x00010203
|
||||
.long 0x04050607
|
||||
.long 0x08090a0b
|
||||
.long 0x0c0d0e0f
|
||||
.long 0x00010203
|
||||
.long 0x04050607
|
||||
.long 0x08090a0b
|
||||
.long 0x0c0d0e0f
|
||||
|
||||
# void sha1_transform(int32_t *hash, const char* input, size_t num_blocks) ;
|
||||
.text
|
||||
ENTRY(sha1_transform)
|
||||
|
||||
push %rbx
|
||||
push %rbp
|
||||
push %r12
|
||||
push %r13
|
||||
push %r14
|
||||
push %r15
|
||||
|
||||
RESERVE_STACK = (W_SIZE*4 + 8+24)
|
||||
|
||||
/* Align stack */
|
||||
mov %rsp, %rbx
|
||||
and $~(0x20-1), %rsp
|
||||
push %rbx
|
||||
sub $RESERVE_STACK, %rsp
|
||||
|
||||
avx2_zeroupper
|
||||
|
||||
lea K_XMM_AR(%rip), K_BASE
|
||||
|
||||
mov CTX, HASH_PTR
|
||||
mov BUF, BUFFER_PTR
|
||||
lea 64(BUF), BUFFER_PTR2
|
||||
|
||||
shl $6, CNT /* mul by 64 */
|
||||
add BUF, CNT
|
||||
add $64, CNT
|
||||
mov CNT, BUFFER_END
|
||||
|
||||
cmp BUFFER_END, BUFFER_PTR2
|
||||
cmovae K_BASE, BUFFER_PTR2
|
||||
|
||||
xmm_mov BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
|
||||
|
||||
SHA1_PIPELINED_MAIN_BODY
|
||||
|
||||
avx2_zeroupper
|
||||
|
||||
add $RESERVE_STACK, %rsp
|
||||
pop %rsp
|
||||
|
||||
pop %r15
|
||||
pop %r14
|
||||
pop %r13
|
||||
pop %r12
|
||||
pop %rbp
|
||||
pop %rbx
|
||||
|
||||
ret
|
||||
|
||||
ENDPROC(sha1_transform)
|
||||
#endif
|
||||
579
pkg/crypto/sha1/sha1_sse3_amd64.asm
Normal file
579
pkg/crypto/sha1/sha1_sse3_amd64.asm
Normal file
@ -0,0 +1,579 @@
|
||||
;---------------------
|
||||
; https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
|
||||
;
|
||||
; License information:
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
; This implementation notably advances the performance of SHA-1 algorithm compared to existing
|
||||
; implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
|
||||
; implementation and are ready to help if issues or concerns arise (you are welcome to leave
|
||||
; a comment or write an email to the authors). It is provided 'as is' and free for either
|
||||
; commercial or non-commercial use.
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;
|
||||
; This code implements two interfaces of SHA-1 update function: 1) working on a single
|
||||
; 64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
|
||||
; version of code is software pipelined and faster overall, it is a default. Assemble
|
||||
; with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
|
||||
;
|
||||
; C++ prototypes of implemented functions are below:
|
||||
;
|
||||
; #ifndef INTEL_SHA1_SINGLEBLOCK
|
||||
; // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
|
||||
; extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
|
||||
; #else
|
||||
; // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
|
||||
; extern "C" void sha1_update_intel(int *hash, const char* input);
|
||||
; #endif
|
||||
;
|
||||
; Function name 'sha1_update_intel' can be changed in the source or via macro:
|
||||
; -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
|
||||
;
|
||||
; It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
|
||||
;
|
||||
; Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
|
||||
; and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
|
||||
; is also required, the default (e.g. one being replaced) function can be provided for
|
||||
; dispatch on such CPUs, the name of old function can be changed in the source or via macro:
|
||||
; -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
|
||||
;
|
||||
; Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
|
||||
;
|
||||
|
||||
%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
|
||||
;; can be replaced with a default SHA-1 update function name
|
||||
%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH sha1_intel_non_ssse3_cpu_stub_
|
||||
%else
|
||||
extern INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
|
||||
%endif
|
||||
|
||||
;; provide alternative SHA-1 update function's name here
|
||||
%ifndef INTEL_SHA1_UPDATE_FUNCNAME
|
||||
%define INTEL_SHA1_UPDATE_FUNCNAME sha1_update_intel
|
||||
%endif
|
||||
|
||||
global INTEL_SHA1_UPDATE_FUNCNAME
|
||||
|
||||
|
||||
%ifndef INTEL_SHA1_SINGLEBLOCK
|
||||
%assign multiblock 1
|
||||
%else
|
||||
%assign multiblock 0
|
||||
%endif
|
||||
|
||||
|
||||
bits 64
|
||||
default rel
|
||||
|
||||
%ifdef WIN_ABI
|
||||
%xdefine arg1 rcx
|
||||
%xdefine arg2 rdx
|
||||
%xdefine arg3 r8
|
||||
%else
|
||||
%xdefine arg1 rdi
|
||||
%xdefine arg2 rsi
|
||||
%xdefine arg3 rdx
|
||||
%endif
|
||||
|
||||
%xdefine ctx arg1
|
||||
%xdefine buf arg2
|
||||
%xdefine cnt arg3
|
||||
|
||||
%macro REGALLOC 0
|
||||
%xdefine A ecx
|
||||
%xdefine B esi
|
||||
%xdefine C edi
|
||||
%xdefine D ebp
|
||||
%xdefine E edx
|
||||
|
||||
%xdefine T1 eax
|
||||
%xdefine T2 ebx
|
||||
%endmacro
|
||||
|
||||
%xdefine K_BASE r8
|
||||
%xdefine HASH_PTR r9
|
||||
%xdefine BUFFER_PTR r10
|
||||
%xdefine BUFFER_END r11
|
||||
|
||||
%xdefine W_TMP xmm0
|
||||
%xdefine W_TMP2 xmm9
|
||||
|
||||
%xdefine W0 xmm1
|
||||
%xdefine W4 xmm2
|
||||
%xdefine W8 xmm3
|
||||
%xdefine W12 xmm4
|
||||
%xdefine W16 xmm5
|
||||
%xdefine W20 xmm6
|
||||
%xdefine W24 xmm7
|
||||
%xdefine W28 xmm8
|
||||
|
||||
%xdefine XMM_SHUFB_BSWAP xmm10
|
||||
|
||||
;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
|
||||
%xdefine WK(t) (rsp + (t & 15)*4)
|
||||
|
||||
;------------------------------------------------------------------------------
|
||||
;
|
||||
; macro implements SHA-1 function's body for single or several 64-byte blocks
|
||||
; first param: function's name
|
||||
; second param: =0 - function implements single 64-byte block hash
|
||||
; =1 - function implements multiple64-byte blocks hash
|
||||
; 3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
|
||||
;
|
||||
%macro SHA1_VECTOR_ASM 2
|
||||
align 4096
|
||||
%1:
|
||||
push rbx
|
||||
push rbp
|
||||
|
||||
%ifdef WIN_ABI
|
||||
push rdi
|
||||
push rsi
|
||||
|
||||
%xdefine stack_size (16*4 + 16*5 + 8)
|
||||
%else
|
||||
%xdefine stack_size (16*4 + 8)
|
||||
%endif
|
||||
|
||||
sub rsp, stack_size
|
||||
|
||||
%ifdef WIN_ABI
|
||||
%xdefine xmm_save_base (rsp + 16*4)
|
||||
|
||||
xmm_mov [xmm_save_base + 0*16], xmm6
|
||||
xmm_mov [xmm_save_base + 1*16], xmm7
|
||||
xmm_mov [xmm_save_base + 2*16], xmm8
|
||||
xmm_mov [xmm_save_base + 3*16], xmm9
|
||||
xmm_mov [xmm_save_base + 4*16], xmm10
|
||||
%endif
|
||||
|
||||
mov HASH_PTR, ctx
|
||||
mov BUFFER_PTR, buf
|
||||
|
||||
%if (%2 == 1)
|
||||
shl cnt, 6 ;; mul by 64
|
||||
add cnt, buf
|
||||
mov BUFFER_END, cnt
|
||||
%endif
|
||||
|
||||
lea K_BASE, [K_XMM_AR]
|
||||
xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
|
||||
|
||||
SHA1_PIPELINED_MAIN_BODY %2
|
||||
|
||||
%ifdef WIN_ABI
|
||||
xmm_mov xmm6, [xmm_save_base + 0*16]
|
||||
xmm_mov xmm7, [xmm_save_base + 1*16]
|
||||
xmm_mov xmm8, [xmm_save_base + 2*16]
|
||||
xmm_mov xmm9, [xmm_save_base + 3*16]
|
||||
xmm_mov xmm10,[xmm_save_base + 4*16]
|
||||
%endif
|
||||
|
||||
add rsp, stack_size
|
||||
|
||||
%ifdef WIN_ABI
|
||||
pop rsi
|
||||
pop rdi
|
||||
%endif
|
||||
|
||||
pop rbp
|
||||
pop rbx
|
||||
|
||||
ret
|
||||
%endmacro
|
||||
|
||||
;--------------------------------------------
|
||||
; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
|
||||
; macro param: =0 - process single 64-byte block
|
||||
; =1 - multiple blocks
|
||||
;
|
||||
%macro SHA1_PIPELINED_MAIN_BODY 1
|
||||
|
||||
REGALLOC
|
||||
|
||||
mov A, [HASH_PTR ]
|
||||
mov B, [HASH_PTR+ 4]
|
||||
mov C, [HASH_PTR+ 8]
|
||||
mov D, [HASH_PTR+12]
|
||||
|
||||
mov E, [HASH_PTR+16]
|
||||
|
||||
%assign i 0
|
||||
%rep W_PRECALC_AHEAD
|
||||
W_PRECALC i
|
||||
%assign i i+1
|
||||
%endrep
|
||||
|
||||
%xdefine F F1
|
||||
|
||||
%if (%1 == 1) ;; code loops through more than one block
|
||||
%%_loop:
|
||||
cmp BUFFER_PTR, K_BASE ;; we use K_BASE value as a signal of a last block,
|
||||
jne %%_begin ;; it is set below by: cmovae BUFFER_PTR, K_BASE
|
||||
jmp %%_end
|
||||
|
||||
align 32
|
||||
%%_begin:
|
||||
%endif
|
||||
RR A,B,C,D,E,0
|
||||
RR D,E,A,B,C,2
|
||||
RR B,C,D,E,A,4
|
||||
RR E,A,B,C,D,6
|
||||
RR C,D,E,A,B,8
|
||||
|
||||
RR A,B,C,D,E,10
|
||||
RR D,E,A,B,C,12
|
||||
RR B,C,D,E,A,14
|
||||
RR E,A,B,C,D,16
|
||||
RR C,D,E,A,B,18
|
||||
|
||||
%xdefine F F2
|
||||
|
||||
RR A,B,C,D,E,20
|
||||
RR D,E,A,B,C,22
|
||||
RR B,C,D,E,A,24
|
||||
RR E,A,B,C,D,26
|
||||
RR C,D,E,A,B,28
|
||||
|
||||
RR A,B,C,D,E,30
|
||||
RR D,E,A,B,C,32
|
||||
RR B,C,D,E,A,34
|
||||
RR E,A,B,C,D,36
|
||||
RR C,D,E,A,B,38
|
||||
|
||||
%xdefine F F3
|
||||
|
||||
RR A,B,C,D,E,40
|
||||
RR D,E,A,B,C,42
|
||||
RR B,C,D,E,A,44
|
||||
RR E,A,B,C,D,46
|
||||
RR C,D,E,A,B,48
|
||||
|
||||
RR A,B,C,D,E,50
|
||||
RR D,E,A,B,C,52
|
||||
RR B,C,D,E,A,54
|
||||
RR E,A,B,C,D,56
|
||||
RR C,D,E,A,B,58
|
||||
|
||||
%xdefine F F4
|
||||
|
||||
%if (%1 == 1) ;; if code loops through more than one block
|
||||
add BUFFER_PTR, 64 ;; move to next 64-byte block
|
||||
cmp BUFFER_PTR, BUFFER_END ;; check if current block is the last one
|
||||
cmovae BUFFER_PTR, K_BASE ;; smart way to signal the last iteration
|
||||
%else
|
||||
%xdefine W_NO_TAIL_PRECALC 1 ;; no software pipelining for single block interface
|
||||
%endif
|
||||
|
||||
RR A,B,C,D,E,60
|
||||
RR D,E,A,B,C,62
|
||||
RR B,C,D,E,A,64
|
||||
RR E,A,B,C,D,66
|
||||
RR C,D,E,A,B,68
|
||||
|
||||
RR A,B,C,D,E,70
|
||||
RR D,E,A,B,C,72
|
||||
RR B,C,D,E,A,74
|
||||
RR E,A,B,C,D,76
|
||||
RR C,D,E,A,B,78
|
||||
|
||||
UPDATE_HASH [HASH_PTR ],A
|
||||
UPDATE_HASH [HASH_PTR+ 4],B
|
||||
UPDATE_HASH [HASH_PTR+ 8],C
|
||||
UPDATE_HASH [HASH_PTR+12],D
|
||||
UPDATE_HASH [HASH_PTR+16],E
|
||||
|
||||
%if (%1 == 1)
|
||||
jmp %%_loop
|
||||
|
||||
align 32
|
||||
%%_end:
|
||||
%endif
|
||||
|
||||
|
||||
%xdefine W_NO_TAIL_PRECALC 0
|
||||
%xdefine F %error
|
||||
|
||||
%endmacro
|
||||
|
||||
|
||||
%macro F1 3
|
||||
mov T1,%2
|
||||
xor T1,%3
|
||||
and T1,%1
|
||||
xor T1,%3
|
||||
%endmacro
|
||||
|
||||
%macro F2 3
|
||||
mov T1,%3
|
||||
xor T1,%2
|
||||
xor T1,%1
|
||||
%endmacro
|
||||
|
||||
%macro F3 3
|
||||
mov T1,%2
|
||||
mov T2,%1
|
||||
or T1,%1
|
||||
and T2,%2
|
||||
and T1,%3
|
||||
or T1,T2
|
||||
%endmacro
|
||||
|
||||
%define F4 F2
|
||||
|
||||
%macro UPDATE_HASH 2
|
||||
add %2, %1
|
||||
mov %1, %2
|
||||
%endmacro
|
||||
|
||||
|
||||
%macro W_PRECALC 1
|
||||
%xdefine i (%1)
|
||||
|
||||
%if (i < 20)
|
||||
%xdefine K_XMM 0
|
||||
%elif (i < 40)
|
||||
%xdefine K_XMM 16
|
||||
%elif (i < 60)
|
||||
%xdefine K_XMM 32
|
||||
%else
|
||||
%xdefine K_XMM 48
|
||||
%endif
|
||||
|
||||
%if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
|
||||
|
||||
%if (W_NO_TAIL_PRECALC == 0)
|
||||
|
||||
%xdefine i ((%1) % 80) ;; pre-compute for the next iteration
|
||||
|
||||
%if (i == 0)
|
||||
W_PRECALC_RESET
|
||||
%endif
|
||||
|
||||
|
||||
W_PRECALC_00_15
|
||||
%endif
|
||||
|
||||
%elif (i < 32)
|
||||
W_PRECALC_16_31
|
||||
%elif (i < 80) ;; rounds 32-79
|
||||
W_PRECALC_32_79
|
||||
%endif
|
||||
%endmacro
|
||||
|
||||
%macro W_PRECALC_RESET 0
|
||||
%xdefine W W0
|
||||
%xdefine W_minus_04 W4
|
||||
%xdefine W_minus_08 W8
|
||||
%xdefine W_minus_12 W12
|
||||
%xdefine W_minus_16 W16
|
||||
%xdefine W_minus_20 W20
|
||||
%xdefine W_minus_24 W24
|
||||
%xdefine W_minus_28 W28
|
||||
%xdefine W_minus_32 W
|
||||
%endmacro
|
||||
|
||||
%macro W_PRECALC_ROTATE 0
|
||||
%xdefine W_minus_32 W_minus_28
|
||||
%xdefine W_minus_28 W_minus_24
|
||||
%xdefine W_minus_24 W_minus_20
|
||||
%xdefine W_minus_20 W_minus_16
|
||||
%xdefine W_minus_16 W_minus_12
|
||||
%xdefine W_minus_12 W_minus_08
|
||||
%xdefine W_minus_08 W_minus_04
|
||||
%xdefine W_minus_04 W
|
||||
%xdefine W W_minus_32
|
||||
%endmacro
|
||||
|
||||
%xdefine W_PRECALC_AHEAD 16
|
||||
%xdefine W_NO_TAIL_PRECALC 0
|
||||
|
||||
|
||||
%xdefine xmm_mov movdqa
|
||||
|
||||
%macro W_PRECALC_00_15 0
|
||||
;; message scheduling pre-compute for rounds 0-15
|
||||
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
||||
movdqu W_TMP, [BUFFER_PTR + (i * 4)]
|
||||
%elif ((i & 3) == 1)
|
||||
pshufb W_TMP, XMM_SHUFB_BSWAP
|
||||
movdqa W, W_TMP
|
||||
%elif ((i & 3) == 2)
|
||||
paddd W_TMP, [K_BASE]
|
||||
%elif ((i & 3) == 3)
|
||||
movdqa [WK(i&~3)], W_TMP
|
||||
|
||||
W_PRECALC_ROTATE
|
||||
%endif
|
||||
%endmacro
|
||||
|
||||
%macro W_PRECALC_16_31 0
|
||||
;; message scheduling pre-compute for rounds 16-31
|
||||
;; calculating last 32 w[i] values in 8 XMM registers
|
||||
;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
|
||||
;;
|
||||
;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
|
||||
;;
|
||||
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
||||
movdqa W, W_minus_12
|
||||
palignr W, W_minus_16, 8 ;; w[i-14]
|
||||
movdqa W_TMP, W_minus_04
|
||||
psrldq W_TMP, 4 ;; w[i-3]
|
||||
pxor W, W_minus_08
|
||||
%elif ((i & 3) == 1)
|
||||
pxor W_TMP, W_minus_16
|
||||
pxor W, W_TMP
|
||||
movdqa W_TMP2, W
|
||||
movdqa W_TMP, W
|
||||
pslldq W_TMP2, 12
|
||||
%elif ((i & 3) == 2)
|
||||
psrld W, 31
|
||||
pslld W_TMP, 1
|
||||
por W_TMP, W
|
||||
movdqa W, W_TMP2
|
||||
psrld W_TMP2, 30
|
||||
pslld W, 2
|
||||
%elif ((i & 3) == 3)
|
||||
pxor W_TMP, W
|
||||
pxor W_TMP, W_TMP2
|
||||
movdqa W, W_TMP
|
||||
paddd W_TMP, [K_BASE + K_XMM]
|
||||
movdqa [WK(i&~3)],W_TMP
|
||||
|
||||
W_PRECALC_ROTATE
|
||||
%endif
|
||||
%endmacro
|
||||
|
||||
%macro W_PRECALC_32_79 0
|
||||
;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1
|
||||
;; instead we do equal: w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
|
||||
;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
|
||||
;;
|
||||
%if ((i & 3) == 0) ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
|
||||
movdqa W_TMP, W_minus_04
|
||||
pxor W, W_minus_28 ;; W is W_minus_32 before xor
|
||||
palignr W_TMP, W_minus_08, 8
|
||||
%elif ((i & 3) == 1)
|
||||
pxor W, W_minus_16
|
||||
pxor W, W_TMP
|
||||
movdqa W_TMP, W
|
||||
%elif ((i & 3) == 2)
|
||||
psrld W, 30
|
||||
pslld W_TMP, 2
|
||||
por W_TMP, W
|
||||
%elif ((i & 3) == 3)
|
||||
movdqa W, W_TMP
|
||||
paddd W_TMP, [K_BASE + K_XMM]
|
||||
movdqa [WK(i&~3)],W_TMP
|
||||
|
||||
W_PRECALC_ROTATE
|
||||
%endif
|
||||
%endmacro
|
||||
|
||||
%macro RR 6 ;; RR does two rounds of SHA-1 back to back with W pre-calculation
|
||||
|
||||
;; TEMP = A
|
||||
;; A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
|
||||
;; C = ROTATE_LEFT( B, 30 )
|
||||
;; D = C
|
||||
;; E = D
|
||||
;; B = TEMP
|
||||
|
||||
W_PRECALC (%6 + W_PRECALC_AHEAD)
|
||||
F %2, %3, %4 ;; F returns result in T1
|
||||
add %5, [WK(%6)]
|
||||
rol %2, 30
|
||||
mov T2, %1
|
||||
add %4, [WK(%6 + 1)]
|
||||
rol T2, 5
|
||||
add %5, T1
|
||||
|
||||
W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
|
||||
add T2, %5
|
||||
mov %5, T2
|
||||
rol T2, 5
|
||||
add %4, T2
|
||||
F %1, %2, %3 ;; F returns result in T1
|
||||
add %4, T1
|
||||
rol %1, 30
|
||||
|
||||
;; write: %1, %2
|
||||
;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
|
||||
%endmacro
|
||||
|
||||
|
||||
|
||||
;;----------------------
|
||||
section .data align=128
|
||||
|
||||
%xdefine K1 0x5a827999
|
||||
%xdefine K2 0x6ed9eba1
|
||||
%xdefine K3 0x8f1bbcdc
|
||||
%xdefine K4 0xca62c1d6
|
||||
|
||||
align 128
|
||||
K_XMM_AR:
|
||||
DD K1, K1, K1, K1
|
||||
DD K2, K2, K2, K2
|
||||
DD K3, K3, K3, K3
|
||||
DD K4, K4, K4, K4
|
||||
|
||||
align 16
|
||||
bswap_shufb_ctl:
|
||||
DD 00010203h
|
||||
DD 04050607h
|
||||
DD 08090a0bh
|
||||
DD 0c0d0e0fh
|
||||
|
||||
;; dispatch pointer, points to the init routine for the first invocation
|
||||
sha1_update_intel_dispatched:
|
||||
DQ sha1_update_intel_init_
|
||||
|
||||
;;----------------------
|
||||
section .text align=4096
|
||||
|
||||
SHA1_VECTOR_ASM sha1_update_intel_ssse3_, multiblock
|
||||
|
||||
align 32
|
||||
sha1_update_intel_init_: ;; we get here with the first time invocation
|
||||
call sha1_update_intel_dispacth_init_
|
||||
INTEL_SHA1_UPDATE_FUNCNAME: ;; we get here after init
|
||||
jmp qword [sha1_update_intel_dispatched]
|
||||
|
||||
;; CPUID feature flag based dispatch
|
||||
sha1_update_intel_dispacth_init_:
|
||||
push rax
|
||||
push rbx
|
||||
push rcx
|
||||
push rdx
|
||||
push rsi
|
||||
|
||||
lea rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
|
||||
|
||||
mov eax, 1
|
||||
cpuid
|
||||
|
||||
test ecx, 0200h ;; SSSE3 support, CPUID.1.ECX[bit 9]
|
||||
jz _done
|
||||
|
||||
lea rsi, [sha1_update_intel_ssse3_]
|
||||
|
||||
_done:
|
||||
mov [sha1_update_intel_dispatched], rsi
|
||||
|
||||
pop rsi
|
||||
pop rdx
|
||||
pop rcx
|
||||
pop rbx
|
||||
pop rax
|
||||
ret
|
||||
|
||||
;;----------------------
|
||||
;; in the case a default SHA-1 update function implementation was not provided
|
||||
;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
|
||||
;; failure in a safest way - jumps to the stub function with UD2 instruction below
|
||||
sha1_intel_non_ssse3_cpu_stub_:
|
||||
ud2 ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
|
||||
ret
|
||||
|
||||
; END
|
||||
;----------------------
|
||||
154
pkg/crypto/sha1/sha1_test.go
Normal file
154
pkg/crypto/sha1/sha1_test.go
Normal file
@ -0,0 +1,154 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file of
|
||||
// Golang project:
|
||||
// https://github.com/golang/go/blob/master/LICENSE
|
||||
|
||||
// Using this part of Minio codebase under the license
|
||||
// Apache License Version 2.0 with modifications
|
||||
|
||||
// SHA1 hash algorithm. See RFC 3174.
|
||||
|
||||
package sha1
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type sha1Test struct {
|
||||
out string
|
||||
in string
|
||||
}
|
||||
|
||||
var golden = []sha1Test{
|
||||
{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""},
|
||||
{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"},
|
||||
{"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"},
|
||||
{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"},
|
||||
{"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"},
|
||||
{"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"},
|
||||
{"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"},
|
||||
{"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"},
|
||||
{"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"},
|
||||
{"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"},
|
||||
{"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"},
|
||||
{"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."},
|
||||
{"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."},
|
||||
{"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."},
|
||||
{"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
|
||||
{"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered. -Tom Stoppard"},
|
||||
{"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."},
|
||||
{"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."},
|
||||
{"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."},
|
||||
{"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
|
||||
{"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
|
||||
{"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size: a.out: bad magic"},
|
||||
{"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail. -Mark Horton"},
|
||||
{"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world. CCFestoon"},
|
||||
{"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."},
|
||||
{"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."},
|
||||
{"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."},
|
||||
{"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"},
|
||||
{"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
|
||||
{"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"},
|
||||
{"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++? -Paul Glick"},
|
||||
}
|
||||
|
||||
func TestGolden(t *testing.T) {
|
||||
for i := 0; i < len(golden); i++ {
|
||||
g := golden[i]
|
||||
s := fmt.Sprintf("%x", Sum([]byte(g.in)))
|
||||
if s != g.out {
|
||||
t.Fatalf("Sum function: sha1(%s) = %s want %s", g.in, s, g.out)
|
||||
}
|
||||
c := New()
|
||||
for j := 0; j < 3; j++ {
|
||||
if j < 2 {
|
||||
io.WriteString(c, g.in)
|
||||
} else {
|
||||
io.WriteString(c, g.in[0:len(g.in)/2])
|
||||
c.Sum(nil)
|
||||
io.WriteString(c, g.in[len(g.in)/2:])
|
||||
}
|
||||
s := fmt.Sprintf("%x", c.Sum(nil))
|
||||
if s != g.out {
|
||||
t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out)
|
||||
}
|
||||
c.Reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSize(t *testing.T) {
|
||||
c := New()
|
||||
if got := c.Size(); got != Size {
|
||||
t.Errorf("Size = %d; want %d", got, Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockSize(t *testing.T) {
|
||||
c := New()
|
||||
if got := c.BlockSize(); got != BlockSize {
|
||||
t.Errorf("BlockSize = %d; want %d", got, BlockSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that blockGeneric (pure Go) and block (in assembly for amd64, 386, arm) match.
|
||||
func TestBlockGeneric(t *testing.T) {
|
||||
gen, asm := New().(*digest), New().(*digest)
|
||||
buf := make([]byte, BlockSize*20) // arbitrary factor
|
||||
rand.Read(buf)
|
||||
blockGeneric(gen, buf)
|
||||
block(asm, buf)
|
||||
if *gen != *asm {
|
||||
t.Error("block and blockGeneric resulted in different states")
|
||||
}
|
||||
}
|
||||
|
||||
var bench = New()
|
||||
var buf = make([]byte, 1024*1024)
|
||||
|
||||
func benchmarkSize(b *testing.B, size int) {
|
||||
b.SetBytes(int64(size))
|
||||
sum := make([]byte, bench.Size())
|
||||
for i := 0; i < b.N; i++ {
|
||||
bench.Reset()
|
||||
bench.Write(buf[:size])
|
||||
bench.Sum(sum[:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkHash8Bytes(b *testing.B) {
|
||||
benchmarkSize(b, 8)
|
||||
}
|
||||
|
||||
func BenchmarkHash1K(b *testing.B) {
|
||||
benchmarkSize(b, 1024)
|
||||
}
|
||||
|
||||
func BenchmarkHash8K(b *testing.B) {
|
||||
benchmarkSize(b, 8192)
|
||||
}
|
||||
|
||||
func BenchmarkHash1M(b *testing.B) {
|
||||
benchmarkSize(b, 1024*1024)
|
||||
}
|
||||
21
pkg/crypto/sha1/sha1_yasm_darwin.go
Normal file
21
pkg/crypto/sha1/sha1_yasm_darwin.go
Normal file
@ -0,0 +1,21 @@
|
||||
// +build darwin,amd64
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
//go:generate yasm -f macho64 -DINTEL_SHA1_UPDATE_FUNCNAME=_sha1_update_intel sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso
|
||||
21
pkg/crypto/sha1/sha1_yasm_linux.go
Normal file
21
pkg/crypto/sha1/sha1_yasm_linux.go
Normal file
@ -0,0 +1,21 @@
|
||||
// +build linux,amd64
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
//go:generate yasm -f elf64 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso
|
||||
21
pkg/crypto/sha1/sha1_yasm_windows.go
Normal file
21
pkg/crypto/sha1/sha1_yasm_windows.go
Normal file
@ -0,0 +1,21 @@
|
||||
// +build windows,amd64
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
//go:generate yasm -f win64 -DWIN_ABI=1 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso
|
||||
43
pkg/crypto/sha1/sha1block.go
Normal file
43
pkg/crypto/sha1/sha1block.go
Normal file
@ -0,0 +1,43 @@
|
||||
// +build amd64,cgo
|
||||
// +build darwin windows
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
// #include <stdint.h>
|
||||
// #include <stdlib.h>
|
||||
// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
|
||||
import "C"
|
||||
import (
|
||||
"unsafe"
|
||||
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
)
|
||||
|
||||
func block(dig *digest, p []byte) {
|
||||
switch true {
|
||||
case cpu.HasSSE41() == true:
|
||||
blockSSE3(dig, p)
|
||||
default:
|
||||
blockGeneric(dig, p)
|
||||
}
|
||||
}
|
||||
|
||||
func blockSSE3(dig *digest, p []byte) {
|
||||
C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
|
||||
}
|
||||
110
pkg/crypto/sha1/sha1block_generic.go
Normal file
110
pkg/crypto/sha1/sha1block_generic.go
Normal file
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file of
|
||||
// Golang project:
|
||||
// https://github.com/golang/go/blob/master/LICENSE
|
||||
|
||||
// Using this part of Minio codebase under the license
|
||||
// Apache License Version 2.0 with modifications
|
||||
|
||||
package sha1
|
||||
|
||||
const (
|
||||
_K0 = 0x5A827999
|
||||
_K1 = 0x6ED9EBA1
|
||||
_K2 = 0x8F1BBCDC
|
||||
_K3 = 0xCA62C1D6
|
||||
)
|
||||
|
||||
// blockGeneric is a portable, pure Go version of the SHA1 block step.
|
||||
// It's used by sha1block_generic.go and tests.
|
||||
func blockGeneric(dig *digest, p []byte) {
|
||||
var w [16]uint32
|
||||
|
||||
h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4]
|
||||
for len(p) >= chunk {
|
||||
// Can interlace the computation of w with the
|
||||
// rounds below if needed for speed.
|
||||
for i := 0; i < 16; i++ {
|
||||
j := i * 4
|
||||
w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
|
||||
}
|
||||
|
||||
a, b, c, d, e := h0, h1, h2, h3, h4
|
||||
|
||||
// Each of the four 20-iteration rounds
|
||||
// differs only in the computation of f and
|
||||
// the choice of K (_K0, _K1, etc).
|
||||
i := 0
|
||||
for ; i < 16; i++ {
|
||||
f := b&c | (^b)&d
|
||||
a5 := a<<5 | a>>(32-5)
|
||||
b30 := b<<30 | b>>(32-30)
|
||||
t := a5 + f + e + w[i&0xf] + _K0
|
||||
a, b, c, d, e = t, a, b30, c, d
|
||||
}
|
||||
for ; i < 20; i++ {
|
||||
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
|
||||
w[i&0xf] = tmp<<1 | tmp>>(32-1)
|
||||
|
||||
f := b&c | (^b)&d
|
||||
a5 := a<<5 | a>>(32-5)
|
||||
b30 := b<<30 | b>>(32-30)
|
||||
t := a5 + f + e + w[i&0xf] + _K0
|
||||
a, b, c, d, e = t, a, b30, c, d
|
||||
}
|
||||
for ; i < 40; i++ {
|
||||
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
|
||||
w[i&0xf] = tmp<<1 | tmp>>(32-1)
|
||||
f := b ^ c ^ d
|
||||
a5 := a<<5 | a>>(32-5)
|
||||
b30 := b<<30 | b>>(32-30)
|
||||
t := a5 + f + e + w[i&0xf] + _K1
|
||||
a, b, c, d, e = t, a, b30, c, d
|
||||
}
|
||||
for ; i < 60; i++ {
|
||||
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
|
||||
w[i&0xf] = tmp<<1 | tmp>>(32-1)
|
||||
f := ((b | c) & d) | (b & c)
|
||||
|
||||
a5 := a<<5 | a>>(32-5)
|
||||
b30 := b<<30 | b>>(32-30)
|
||||
t := a5 + f + e + w[i&0xf] + _K2
|
||||
a, b, c, d, e = t, a, b30, c, d
|
||||
}
|
||||
for ; i < 80; i++ {
|
||||
tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
|
||||
w[i&0xf] = tmp<<1 | tmp>>(32-1)
|
||||
f := b ^ c ^ d
|
||||
a5 := a<<5 | a>>(32-5)
|
||||
b30 := b<<30 | b>>(32-30)
|
||||
t := a5 + f + e + w[i&0xf] + _K3
|
||||
a, b, c, d, e = t, a, b30, c, d
|
||||
}
|
||||
|
||||
h0 += a
|
||||
h1 += b
|
||||
h2 += c
|
||||
h3 += d
|
||||
h4 += e
|
||||
|
||||
p = p[chunk:]
|
||||
}
|
||||
dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4
|
||||
}
|
||||
50
pkg/crypto/sha1/sha1block_linux.go
Normal file
50
pkg/crypto/sha1/sha1block_linux.go
Normal file
@ -0,0 +1,50 @@
|
||||
// +build linux,amd64,cgo
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
// #cgo CFLAGS: -DHAS_AVX2
|
||||
// #include <stdint.h>
|
||||
// #include <stdlib.h>
|
||||
// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
|
||||
// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
|
||||
import "C"
|
||||
import (
|
||||
"unsafe"
|
||||
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
)
|
||||
|
||||
func block(dig *digest, p []byte) {
|
||||
switch true {
|
||||
case cpu.HasAVX2():
|
||||
blockAVX2(dig, p)
|
||||
case cpu.HasSSE41():
|
||||
blockSSE3(dig, p)
|
||||
default:
|
||||
blockGeneric(dig, p)
|
||||
}
|
||||
}
|
||||
|
||||
func blockAVX2(dig *digest, p []byte) {
|
||||
C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
|
||||
}
|
||||
|
||||
func blockSSE3(dig *digest, p []byte) {
|
||||
C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
|
||||
}
|
||||
23
pkg/crypto/sha1/sha1block_nocgo.go
Normal file
23
pkg/crypto/sha1/sha1block_nocgo.go
Normal file
@ -0,0 +1,23 @@
|
||||
// +build !cgo arm
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package sha1
|
||||
|
||||
func block(dig *digest, p []byte) {
|
||||
blockGeneric(dig, p)
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
// +build 386 arm amd64,!cgo
|
||||
// +build darwin windows 386 arm !cgo
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -20,36 +20,22 @@ package sha256
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"crypto/sha256"
|
||||
)
|
||||
|
||||
// Sum256 - single caller sha256 helper
|
||||
func Sum256(data []byte) []byte {
|
||||
d := sha256.New()
|
||||
d.Write(data)
|
||||
return d.Sum(nil)
|
||||
}
|
||||
// Size - The size of a SHA256 checksum in bytes.
|
||||
const Size = 32
|
||||
|
||||
// Sum - io.Reader based streaming sha256 helper
|
||||
func Sum(reader io.Reader) ([]byte, error) {
|
||||
d := sha256.New()
|
||||
var err error
|
||||
for err == nil {
|
||||
length := 0
|
||||
byteBuffer := make([]byte, 1024*1024)
|
||||
length, err = reader.Read(byteBuffer)
|
||||
byteBuffer = byteBuffer[0:length]
|
||||
d.Write(byteBuffer)
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
return d.Sum(nil), nil
|
||||
}
|
||||
// BlockSize - The blocksize of SHA256 in bytes.
|
||||
const BlockSize = 64
|
||||
|
||||
// New returns a new hash.Hash computing SHA256.
|
||||
func New() hash.Hash {
|
||||
return sha256.New()
|
||||
}
|
||||
|
||||
// Sum256 - single caller sha256 helper
|
||||
func Sum256(data []byte) [Size]byte {
|
||||
return sha256.Sum256(data)
|
||||
}
|
||||
@ -1,4 +1,20 @@
|
||||
// +build amd64,cgo
|
||||
// +build linux,amd64,cgo
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
@ -14,9 +30,8 @@ package sha256
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/cpu"
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
)
|
||||
|
||||
// Size - The size of a SHA256 checksum in bytes.
|
||||
@ -79,6 +94,14 @@ func New() hash.Hash {
|
||||
return d
|
||||
}
|
||||
|
||||
// Sum256 - single caller sha256 helper
|
||||
func Sum256(data []byte) [Size]byte {
|
||||
var d digest
|
||||
d.Reset()
|
||||
d.Write(data)
|
||||
return d.checkSum()
|
||||
}
|
||||
|
||||
// Return size of checksum
|
||||
func (d *digest) Size() int { return Size }
|
||||
|
||||
@ -152,30 +175,3 @@ func (d *digest) checkSum() [Size]byte {
|
||||
|
||||
return digest
|
||||
}
|
||||
|
||||
/// Convenience functions
|
||||
|
||||
// Sum256 - single caller sha256 helper
|
||||
func Sum256(data []byte) []byte {
|
||||
var d digest
|
||||
d.Reset()
|
||||
d.Write(data)
|
||||
return d.Sum(nil)
|
||||
}
|
||||
|
||||
// Sum - io.Reader based streaming sha256 helper
|
||||
func Sum(reader io.Reader) ([]byte, error) {
|
||||
h := New()
|
||||
var err error
|
||||
for err == nil {
|
||||
length := 0
|
||||
byteBuffer := make([]byte, 1024*1024)
|
||||
length, err = reader.Read(byteBuffer)
|
||||
byteBuffer = byteBuffer[0:length]
|
||||
h.Write(byteBuffer)
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
return h.Sum(nil), nil
|
||||
}
|
||||
@ -1,4 +1,18 @@
|
||||
// +build amd64,cgo
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
@ -1,4 +1,4 @@
|
||||
// +build amd64,cgo
|
||||
// +build linux,amd64,cgo
|
||||
|
||||
//
|
||||
// Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
@ -1,7 +1,7 @@
|
||||
// +build 386 arm amd64,!cgo
|
||||
// +build darwin windows 386 arm !cgo
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@ -20,41 +20,22 @@ package sha512
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"crypto/sha512"
|
||||
)
|
||||
|
||||
// The size of a SHA512 checksum in bytes.
|
||||
const (
|
||||
Size = sha512.Size
|
||||
)
|
||||
// Size - The size of a SHA512 checksum in bytes.
|
||||
const Size = 64
|
||||
|
||||
// Sum512 - single caller sha512 helper
|
||||
func Sum512(data []byte) []byte {
|
||||
d := sha512.New()
|
||||
d.Write(data)
|
||||
return d.Sum(nil)
|
||||
}
|
||||
|
||||
// Sum - io.Reader based streaming sha512 helper
|
||||
func Sum(reader io.Reader) ([]byte, error) {
|
||||
d := sha512.New()
|
||||
var err error
|
||||
for err == nil {
|
||||
length := 0
|
||||
byteBuffer := make([]byte, 1024*1024)
|
||||
length, err = reader.Read(byteBuffer)
|
||||
byteBuffer = byteBuffer[0:length]
|
||||
d.Write(byteBuffer)
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
return d.Sum(nil), nil
|
||||
}
|
||||
// BlockSize - The blocksize of SHA512 in bytes.
|
||||
const BlockSize = 128
|
||||
|
||||
// New returns a new hash.Hash computing SHA512.
|
||||
func New() hash.Hash {
|
||||
return sha512.New()
|
||||
}
|
||||
|
||||
// Sum512 - single caller sha512 helper
|
||||
func Sum512(data []byte) [Size]byte {
|
||||
return sha512.Sum512(data)
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
// +build amd64,cgo
|
||||
// +build linux,amd64,cgo
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
@ -15,9 +15,8 @@ package sha512
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/cpu"
|
||||
"github.com/minio/minio/pkg/cpu"
|
||||
)
|
||||
|
||||
// Size - The size of a SHA512 checksum in bytes.
|
||||
@ -80,6 +79,14 @@ func New() hash.Hash {
|
||||
return d
|
||||
}
|
||||
|
||||
// Sum512 - single caller sha512 helper
|
||||
func Sum512(data []byte) [Size]byte {
|
||||
var d digest
|
||||
d.Reset()
|
||||
d.Write(data)
|
||||
return d.checkSum()
|
||||
}
|
||||
|
||||
// Return output array byte size
|
||||
func (d *digest) Size() int { return Size }
|
||||
|
||||
@ -157,30 +164,3 @@ func (d *digest) checkSum() [Size]byte {
|
||||
|
||||
return digest
|
||||
}
|
||||
|
||||
/// Convenience functions
|
||||
|
||||
// Sum512 - single caller sha512 helper
|
||||
func Sum512(data []byte) []byte {
|
||||
var d digest
|
||||
d.Reset()
|
||||
d.Write(data)
|
||||
return d.Sum(nil)
|
||||
}
|
||||
|
||||
// Sum - io.Reader based streaming sha512 helper
|
||||
func Sum(reader io.Reader) ([]byte, error) {
|
||||
h := New()
|
||||
var err error
|
||||
for err == nil {
|
||||
length := 0
|
||||
byteBuffer := make([]byte, 1024*1024)
|
||||
length, err = reader.Read(byteBuffer)
|
||||
byteBuffer = byteBuffer[0:length]
|
||||
h.Write(byteBuffer)
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
return h.Sum(nil), nil
|
||||
}
|
||||
@ -1,4 +1,18 @@
|
||||
// +build amd64,cgo
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
@ -1,20 +1,21 @@
|
||||
// +build amd64,cgo
|
||||
// +build linux,amd64,cgo
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
//
|
||||
// Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Software block transform are provided by The Go Authors:
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
1
pkg/erasure/.gitignore
vendored
Normal file
1
pkg/erasure/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.syso
|
||||
64
pkg/erasure/INSTALLGO.md
Normal file
64
pkg/erasure/INSTALLGO.md
Normal file
@ -0,0 +1,64 @@
|
||||
## Ubuntu (Kylin) 14.04
|
||||
### Build Dependencies
|
||||
This installation document assumes Ubuntu 14.04+ on x86-64 platform.
|
||||
|
||||
##### Install Git, GCC, yasm
|
||||
```sh
|
||||
$ sudo apt-get install git build-essential yasm
|
||||
```
|
||||
|
||||
##### Install Go 1.5.1+
|
||||
|
||||
Download Go 1.5.1+ from [https://golang.org/dl/](https://golang.org/dl/).
|
||||
|
||||
```sh
|
||||
$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
|
||||
$ mkdir -p ${HOME}/bin/
|
||||
$ mkdir -p ${HOME}/go/
|
||||
$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
|
||||
```
|
||||
##### Setup GOROOT and GOPATH
|
||||
|
||||
Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
|
||||
and GOPATH specifies the location of your project workspace.
|
||||
|
||||
```sh
|
||||
$ export GOROOT=${HOME}/bin/go
|
||||
$ export GOPATH=${HOME}/go
|
||||
$ export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
|
||||
```
|
||||
|
||||
## OS X (Yosemite) 10.10
|
||||
### Build Dependencies
|
||||
This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
|
||||
|
||||
##### Install brew
|
||||
```sh
|
||||
$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
|
||||
```
|
||||
|
||||
##### Install Git, Python
|
||||
```sh
|
||||
$ brew install git python yasm
|
||||
```
|
||||
|
||||
##### Install Go 1.5.1+
|
||||
|
||||
Install golang binaries using `brew`
|
||||
|
||||
```sh
|
||||
$ brew install go
|
||||
$ mkdir -p $HOME/go
|
||||
```
|
||||
|
||||
##### Setup GOROOT and GOPATH
|
||||
|
||||
Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
|
||||
and GOPATH specifies the location of your project workspace.
|
||||
|
||||
```sh
|
||||
$ export GOPATH=${HOME}/go
|
||||
$ export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
|
||||
$ export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
|
||||
$ export PATH=$PATH:${GOPATH}/bin
|
||||
```
|
||||
26
pkg/erasure/LICENSE.INTEL
Normal file
26
pkg/erasure/LICENSE.INTEL
Normal file
@ -0,0 +1,26 @@
|
||||
Copyright(c) 2011-2014 Intel Corporation All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in
|
||||
the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Intel Corporation nor the names of its
|
||||
contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
202
pkg/erasure/LICENSE.MINIO
Normal file
202
pkg/erasure/LICENSE.MINIO
Normal file
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
25
pkg/erasure/README.md
Normal file
25
pkg/erasure/README.md
Normal file
@ -0,0 +1,25 @@
|
||||
## Introduction
|
||||
|
||||
Erasure is an open source Golang library written on top of ISAL (Intel Intelligent Storage Library) released under [Apache license v2](./LICENSE)
|
||||
|
||||
### Developers
|
||||
* [Get Source](./CONTRIBUTING.md)
|
||||
* [Build Dependencies](./BUILDDEPS.md)
|
||||
* [Development Workflow](./CONTRIBUTING.md#developer-guidelines)
|
||||
* [Developer discussions and bugs](https://github.com/minio/minio/issues)
|
||||
|
||||
### Supported platforms
|
||||
|
||||
| Name | Supported |
|
||||
| ------------- | ------------- |
|
||||
| Linux | Yes |
|
||||
| Windows | Not yet |
|
||||
| Mac OSX | Yes |
|
||||
|
||||
### Supported architectures
|
||||
|
||||
| Arch | Supported |
|
||||
| ------------- | ------------- |
|
||||
| x86-64 | Yes |
|
||||
| arm64 | Not yet|
|
||||
| i386 | Never |
|
||||
62
pkg/erasure/ctypes.go
Normal file
62
pkg/erasure/ctypes.go
Normal file
@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
// #include <stdint.h>
|
||||
import "C"
|
||||
import (
|
||||
"fmt"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// intSlice2CIntArray converts Go int slice to C int array
|
||||
func intSlice2CIntArray(srcErrList []int) *C.int32_t {
|
||||
if len(srcErrList) == 0 {
|
||||
return (*C.int32_t)(unsafe.Pointer(nil))
|
||||
}
|
||||
var sizeErrInt = int(unsafe.Sizeof(srcErrList[0]))
|
||||
switch sizeInt {
|
||||
case sizeErrInt:
|
||||
return (*C.int32_t)(unsafe.Pointer(&srcErrList[0]))
|
||||
case sizeInt8:
|
||||
int8Array := make([]int8, len(srcErrList))
|
||||
for i, v := range srcErrList {
|
||||
int8Array[i] = int8(v)
|
||||
}
|
||||
return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
|
||||
case sizeInt16:
|
||||
int16Array := make([]int16, len(srcErrList))
|
||||
for i, v := range srcErrList {
|
||||
int16Array[i] = int16(v)
|
||||
}
|
||||
return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
|
||||
case sizeInt32:
|
||||
int32Array := make([]int32, len(srcErrList))
|
||||
for i, v := range srcErrList {
|
||||
int32Array[i] = int32(v)
|
||||
}
|
||||
return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
|
||||
case sizeInt64:
|
||||
int64Array := make([]int64, len(srcErrList))
|
||||
for i, v := range srcErrList {
|
||||
int64Array[i] = int64(v)
|
||||
}
|
||||
return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
|
||||
default:
|
||||
panic(fmt.Sprintf("Unsupported: %d", sizeInt))
|
||||
}
|
||||
}
|
||||
66
pkg/erasure/doc.go
Normal file
66
pkg/erasure/doc.go
Normal file
@ -0,0 +1,66 @@
|
||||
// Package erasure is a Go wrapper for the Intel Intelligent Storage
|
||||
// Acceleration Library (Intel ISA-L). Intel ISA-L is a CPU optimized
|
||||
// implementation of erasure coding algorithms.
|
||||
//
|
||||
// For more information on Intel ISA-L, please visit:
|
||||
// https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// Encode encodes a block of data. The input is the original data. The output
|
||||
// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
|
||||
// length of the original object.
|
||||
//
|
||||
// Decode decodes 2 tuple data containing (k + m) chunks back into its original form.
|
||||
// Additionally original block length should also be provided as input.
|
||||
//
|
||||
// Decoded data is exactly similar in length and content as the original data.
|
||||
//
|
||||
// Encoding data may be performed in 3 steps.
|
||||
//
|
||||
// 1. Create a parse set of encoder parameters
|
||||
// 2. Create a new encoder
|
||||
// 3. Encode data
|
||||
//
|
||||
// Decoding data is also performed in 3 steps.
|
||||
//
|
||||
// 1. Create a parse set of encoder parameters for validation
|
||||
// 2. Create a new encoder
|
||||
// 3. Decode data
|
||||
//
|
||||
// Erasure parameters contain three configurable elements:
|
||||
// ValidateParams(k, m, technique int) (ErasureParams, error)
|
||||
// k - Number of rows in matrix
|
||||
// m - Number of colums in matrix
|
||||
// technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
|
||||
// constraints: k + m < Galois Field (2^8)
|
||||
//
|
||||
// Choosing right parity and matrix technique is left for application to decide.
|
||||
//
|
||||
// But here are the few points to keep in mind
|
||||
//
|
||||
// Matrix Type:
|
||||
// - Vandermonde is most commonly used method for choosing coefficients in erasure
|
||||
// encoding but does not guarantee invertable for every sub matrix.
|
||||
// - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding.
|
||||
// Since any sub-matrix of a Cauchy matrix is invertable.
|
||||
//
|
||||
// Total blocks:
|
||||
// - Data blocks and Parity blocks should not be greater than 'Galois Field' (2^8)
|
||||
//
|
||||
// Example
|
||||
//
|
||||
// Creating and using an encoder
|
||||
// var bytes []byte
|
||||
// params := erasure.ValidateParams(10, 5)
|
||||
// encoder := erasure.NewErasure(params)
|
||||
// encodedData, length := encoder.Encode(bytes)
|
||||
//
|
||||
// Creating and using a decoder
|
||||
// var encodedData [][]byte
|
||||
// var length int
|
||||
// params := erasure.ValidateParams(10, 5)
|
||||
// encoder := erasure.NewErasure(params)
|
||||
// originalData, err := encoder.Decode(encodedData, length)
|
||||
//
|
||||
package erasure
|
||||
39
pkg/erasure/ec_minio_common.h
Normal file
39
pkg/erasure/ec_minio_common.h
Normal file
@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef __COMMON_H__
|
||||
#define __COMMON_H__
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
int32_t minio_init_encoder (int k, int m,
|
||||
unsigned char **encode_matrix,
|
||||
unsigned char **encode_tbls);
|
||||
|
||||
int32_t minio_init_decoder (int32_t *error_index,
|
||||
int k, int n, int errs,
|
||||
unsigned char *encoding_matrix,
|
||||
unsigned char **decode_matrix,
|
||||
unsigned char **decode_tbls,
|
||||
uint32_t **decode_index);
|
||||
|
||||
int32_t minio_get_source_target (int errs, int k, int m,
|
||||
int32_t *error_index,
|
||||
uint32_t *decode_index,
|
||||
unsigned char **buffs,
|
||||
unsigned char ***source,
|
||||
unsigned char ***target);
|
||||
#endif /* __COMMON_H__ */
|
||||
142
pkg/erasure/ec_minio_decode.c
Normal file
142
pkg/erasure/ec_minio_decode.c
Normal file
@ -0,0 +1,142 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "isa-l.h"
|
||||
#include "ec_minio_common.h"
|
||||
|
||||
static
|
||||
int32_t _minio_src_index_in_error (int r, int32_t *error_index, int errs)
|
||||
{
|
||||
int i;
|
||||
for (i = 0; i < errs; i++) {
|
||||
if (error_index[i] == r) {
|
||||
// true
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
// false
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Separate out source data and target buffers
|
||||
int32_t minio_get_source_target (int errs, int k, int m,
|
||||
int32_t *error_index,
|
||||
uint32_t *decode_index,
|
||||
unsigned char **buffs,
|
||||
unsigned char ***source,
|
||||
unsigned char ***target)
|
||||
{
|
||||
int i;
|
||||
unsigned char *tmp_source[k];
|
||||
unsigned char *tmp_target[m];
|
||||
|
||||
if (k < 0 || m < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
memset (tmp_source, 0, k);
|
||||
memset (tmp_target, 0, m);
|
||||
|
||||
for (i = 0; i < k; i++) {
|
||||
tmp_source[i] = (unsigned char *) buffs[decode_index[i]];
|
||||
}
|
||||
|
||||
for (i = 0; i < m; i++) {
|
||||
if (i < errs)
|
||||
tmp_target[i] = (unsigned char *) buffs[error_index[i]];
|
||||
}
|
||||
|
||||
*source = tmp_source;
|
||||
*target = tmp_target;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
Generate decode matrix during the decoding phase
|
||||
*/
|
||||
|
||||
int minio_init_decoder (int32_t *error_index,
|
||||
int k, int n, int errs,
|
||||
unsigned char *encode_matrix,
|
||||
unsigned char **decode_matrix,
|
||||
unsigned char **decode_tbls,
|
||||
uint32_t **decode_index)
|
||||
{
|
||||
int i, j, r, l;
|
||||
|
||||
uint32_t *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
|
||||
unsigned char *input_matrix;
|
||||
unsigned char *inverse_matrix;
|
||||
unsigned char *tmp_decode_matrix;
|
||||
unsigned char *tmp_decode_tbls;
|
||||
|
||||
input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
|
||||
inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
|
||||
tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
|
||||
tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);
|
||||
|
||||
for (i = 0, r = 0; i < k; i++, r++) {
|
||||
while (_minio_src_index_in_error(r, error_index, errs))
|
||||
r++;
|
||||
for (j = 0; j < k; j++) {
|
||||
input_matrix[k * i + j] = encode_matrix[k * r + j];
|
||||
}
|
||||
tmp_decode_index[i] = r;
|
||||
}
|
||||
|
||||
// Not all vandermonde matrix can be inverted
|
||||
if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
|
||||
free(tmp_decode_matrix);
|
||||
free(tmp_decode_tbls);
|
||||
free(tmp_decode_index);
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (l = 0; l < errs; l++) {
|
||||
if (error_index[l] < k) {
|
||||
// decoding matrix elements for data chunks
|
||||
for (j = 0; j < k; j++) {
|
||||
tmp_decode_matrix[k * l + j] =
|
||||
inverse_matrix[k *
|
||||
error_index[l] + j];
|
||||
}
|
||||
} else {
|
||||
// decoding matrix element for coding chunks
|
||||
for (i = 0; i < k; i++) {
|
||||
unsigned char s = 0;
|
||||
for (j = 0; j < k; j++) {
|
||||
s ^= gf_mul(inverse_matrix[j * k + i],
|
||||
encode_matrix[k *
|
||||
error_index[l] + j]);
|
||||
}
|
||||
tmp_decode_matrix[k * l + i] = s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
|
||||
|
||||
*decode_matrix = tmp_decode_matrix;
|
||||
*decode_tbls = tmp_decode_tbls;
|
||||
*decode_index = tmp_decode_index;
|
||||
|
||||
return 0;
|
||||
}
|
||||
55
pkg/erasure/ec_minio_encode.c
Normal file
55
pkg/erasure/ec_minio_encode.c
Normal file
@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "isa-l.h"
|
||||
#include "ec_minio_common.h"
|
||||
|
||||
/*
|
||||
Generate encode matrix during the encoding phase
|
||||
*/
|
||||
|
||||
int32_t minio_init_encoder (int k, int m, unsigned char **encode_matrix, unsigned char **encode_tbls)
|
||||
{
|
||||
unsigned char *tmp_matrix;
|
||||
unsigned char *tmp_tbls;
|
||||
|
||||
tmp_matrix = (unsigned char *) malloc (k * (k + m));
|
||||
tmp_tbls = (unsigned char *) malloc (k * (k + m) * 32);
|
||||
|
||||
if (k < 5) {
|
||||
/*
|
||||
Commonly used method for choosing coefficients in erasure
|
||||
encoding but does not guarantee invertable for every sub
|
||||
matrix. For large k it is possible to find cases where the
|
||||
decode matrix chosen from sources and parity not in erasure
|
||||
are not invertable. Users may want to adjust for k > 5.
|
||||
-- Intel
|
||||
*/
|
||||
gf_gen_rs_matrix (tmp_matrix, k + m, k);
|
||||
} else {
|
||||
gf_gen_cauchy1_matrix (tmp_matrix, k + m, k);
|
||||
}
|
||||
|
||||
ec_init_tables(k, m, &tmp_matrix[k * k], tmp_tbls);
|
||||
|
||||
*encode_matrix = tmp_matrix;
|
||||
*encode_tbls = tmp_tbls;
|
||||
|
||||
return 0;
|
||||
}
|
||||
123
pkg/erasure/erasure_decode.go
Normal file
123
pkg/erasure/erasure_decode.go
Normal file
@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
// #cgo CFLAGS: -O0
|
||||
// #include <stdlib.h>
|
||||
// #include "isa-l.h"
|
||||
// #include "ec_minio_common.h"
|
||||
import "C"
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Decode decodes erasure coded blocks of data into its original
|
||||
// form. Erasure coded data contains K data blocks and M parity
|
||||
// blocks. Decode can withstand data loss up to any M number of blocks.
|
||||
//
|
||||
// "encodedDataBlocks" is an array of K data blocks and M parity
|
||||
// blocks. Data blocks are position and order dependent. Missing blocks
|
||||
// are set to "nil". There must be at least "K" number of data|parity
|
||||
// blocks.
|
||||
//
|
||||
// "dataLen" is the length of original source data
|
||||
func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
|
||||
e.mutex.Lock()
|
||||
defer e.mutex.Unlock()
|
||||
|
||||
var source, target **C.uchar
|
||||
|
||||
k := int(e.params.K)
|
||||
m := int(e.params.M)
|
||||
n := k + m
|
||||
// We need the data and parity blocks preserved in the same order. Missing blocks are set to nil.
|
||||
if len(encodedDataBlocks) != n {
|
||||
msg := fmt.Sprintf("Encoded data blocks slice must of length [%d]", n)
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
|
||||
// Length of a single encoded block
|
||||
encodedBlockLen := GetEncodedBlockLen(dataLen, uint8(k))
|
||||
|
||||
// Keep track of errors per block.
|
||||
missingEncodedBlocks := make([]int, n+1)
|
||||
var missingEncodedBlocksCount int
|
||||
|
||||
// Check for the missing encoded blocks
|
||||
for i := range encodedDataBlocks {
|
||||
if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
|
||||
missingEncodedBlocks[missingEncodedBlocksCount] = i
|
||||
missingEncodedBlocksCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Cannot reconstruct original data. Need at least M number of data or parity blocks.
|
||||
if missingEncodedBlocksCount > m {
|
||||
return nil, fmt.Errorf("Cannot reconstruct original data. Need at least [%d] data or parity blocks", m)
|
||||
}
|
||||
|
||||
// Convert from Go int slice to C int array
|
||||
missingEncodedBlocksC := intSlice2CIntArray(missingEncodedBlocks[:missingEncodedBlocksCount])
|
||||
|
||||
// Allocate buffer for the missing blocks
|
||||
for i := range encodedDataBlocks {
|
||||
if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
|
||||
encodedDataBlocks[i] = make([]byte, encodedBlockLen)
|
||||
}
|
||||
}
|
||||
|
||||
// If not already initialized, recompute and cache
|
||||
if e.decodeMatrix == nil || e.decodeTbls == nil || e.decodeIndex == nil {
|
||||
var decodeMatrix, decodeTbls *C.uchar
|
||||
var decodeIndex *C.uint32_t
|
||||
|
||||
C.minio_init_decoder(missingEncodedBlocksC, C.int(k), C.int(n), C.int(missingEncodedBlocksCount),
|
||||
e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)
|
||||
|
||||
// cache this for future needs
|
||||
e.decodeMatrix = decodeMatrix
|
||||
e.decodeTbls = decodeTbls
|
||||
e.decodeIndex = decodeIndex
|
||||
}
|
||||
|
||||
// Make a slice of pointers to encoded blocks. Necessary to bridge to the C world.
|
||||
pointers := make([]*byte, n)
|
||||
for i := range encodedDataBlocks {
|
||||
pointers[i] = &encodedDataBlocks[i][0]
|
||||
}
|
||||
|
||||
// Get pointers to source "data" and target "parity" blocks from the output byte array.
|
||||
ret := C.minio_get_source_target(C.int(missingEncodedBlocksCount), C.int(k), C.int(m), missingEncodedBlocksC,
|
||||
e.decodeIndex, (**C.uchar)(unsafe.Pointer(&pointers[0])), &source, &target)
|
||||
if int(ret) == -1 {
|
||||
return nil, errors.New("Unable to decode data")
|
||||
}
|
||||
|
||||
// Decode data
|
||||
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(missingEncodedBlocksCount), e.decodeTbls,
|
||||
source, target)
|
||||
|
||||
// Allocate buffer to output buffer
|
||||
decodedData = make([]byte, 0, encodedBlockLen*int(k))
|
||||
for i := 0; i < int(k); i++ {
|
||||
decodedData = append(decodedData, encodedDataBlocks[i]...)
|
||||
}
|
||||
|
||||
return decodedData[:dataLen], nil
|
||||
}
|
||||
174
pkg/erasure/erasure_encode.go
Normal file
174
pkg/erasure/erasure_encode.go
Normal file
@ -0,0 +1,174 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
// #include <stdlib.h>
|
||||
// #include "isa-l.h"
|
||||
// #include "ec_minio_common.h"
|
||||
import "C"
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Block alignment
|
||||
const (
|
||||
SIMDAlign = 32
|
||||
)
|
||||
|
||||
// Params is a configuration set for building an encoder. It is created using ValidateParams().
|
||||
type Params struct {
|
||||
K uint8
|
||||
M uint8
|
||||
}
|
||||
|
||||
// Erasure is an object used to encode and decode data.
|
||||
type Erasure struct {
|
||||
params *Params
|
||||
encodeMatrix, encodeTbls *C.uchar
|
||||
decodeMatrix, decodeTbls *C.uchar
|
||||
decodeIndex *C.uint32_t
|
||||
mutex *sync.Mutex
|
||||
}
|
||||
|
||||
// ValidateParams creates an Params object.
|
||||
//
|
||||
// k and m represent the matrix size, which corresponds to the protection level
|
||||
// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
|
||||
//
|
||||
func ValidateParams(k, m uint8) (*Params, error) {
|
||||
if k < 1 {
|
||||
return nil, errors.New("k cannot be zero")
|
||||
}
|
||||
|
||||
if m < 1 {
|
||||
return nil, errors.New("m cannot be zero")
|
||||
}
|
||||
|
||||
if k+m > 255 {
|
||||
return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
|
||||
}
|
||||
|
||||
return &Params{
|
||||
K: k,
|
||||
M: m,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewErasure creates an encoder object with a given set of parameters.
|
||||
func NewErasure(ep *Params) *Erasure {
|
||||
var k = C.int(ep.K)
|
||||
var m = C.int(ep.M)
|
||||
|
||||
var encodeMatrix *C.uchar
|
||||
var encodeTbls *C.uchar
|
||||
|
||||
C.minio_init_encoder(k, m, &encodeMatrix, &encodeTbls)
|
||||
|
||||
return &Erasure{
|
||||
params: ep,
|
||||
encodeMatrix: encodeMatrix,
|
||||
encodeTbls: encodeTbls,
|
||||
decodeMatrix: nil,
|
||||
decodeTbls: nil,
|
||||
decodeIndex: nil,
|
||||
mutex: new(sync.Mutex),
|
||||
}
|
||||
}
|
||||
|
||||
// GetEncodedBlocksLen - total length of all encoded blocks
|
||||
func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
|
||||
outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
|
||||
return outputLen
|
||||
}
|
||||
|
||||
// GetEncodedBlockLen - length per block of encoded blocks
|
||||
func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
|
||||
alignment := int(k) * SIMDAlign
|
||||
remainder := inputLen % alignment
|
||||
|
||||
paddedInputLen := inputLen
|
||||
if remainder != 0 {
|
||||
paddedInputLen = inputLen + (alignment - remainder)
|
||||
}
|
||||
encodedOutputLen = paddedInputLen / int(k)
|
||||
return encodedOutputLen
|
||||
}
|
||||
|
||||
// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
|
||||
// Output is [k+m][]blocks of data and parity slices.
|
||||
func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
|
||||
e.mutex.Lock()
|
||||
defer e.mutex.Unlock()
|
||||
|
||||
k := int(e.params.K) // "k" data blocks
|
||||
m := int(e.params.M) // "m" parity blocks
|
||||
n := k + m // "n" total encoded blocks
|
||||
|
||||
// Length of a single encoded chunk.
|
||||
// Total number of encoded chunks = "k" data + "m" parity blocks
|
||||
encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
|
||||
|
||||
// Length of total number of "k" data chunks
|
||||
encodedDataBlocksLen := encodedBlockLen * k
|
||||
|
||||
// Length of extra padding required for the data blocks.
|
||||
encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
|
||||
|
||||
// Extend inputData buffer to accommodate coded data blocks if necesssary
|
||||
if encodedDataBlocksPadLen > 0 {
|
||||
padding := make([]byte, encodedDataBlocksPadLen)
|
||||
// Expand with new padded blocks to the byte array
|
||||
inputData = append(inputData, padding...)
|
||||
}
|
||||
|
||||
// Extend inputData buffer to accommodate coded parity blocks
|
||||
{ // Local Scope
|
||||
encodedParityBlocksLen := encodedBlockLen * m
|
||||
parityBlocks := make([]byte, encodedParityBlocksLen)
|
||||
inputData = append(inputData, parityBlocks...)
|
||||
}
|
||||
|
||||
// Allocate memory to the "encoded blocks" return buffer
|
||||
encodedBlocks = make([][]byte, n) // Return buffer
|
||||
|
||||
// Necessary to bridge Go to the C world. C requires 2D arry of pointers to
|
||||
// byte array. "encodedBlocks" is a 2D slice.
|
||||
pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
|
||||
|
||||
// Copy data block slices to encoded block buffer
|
||||
for i := 0; i < k; i++ {
|
||||
encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
|
||||
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
|
||||
}
|
||||
|
||||
// Copy erasure block slices to encoded block buffer
|
||||
for i := k; i < n; i++ {
|
||||
encodedBlocks[i] = make([]byte, encodedBlockLen)
|
||||
pointersToEncodedBlock[i] = &encodedBlocks[i][0]
|
||||
}
|
||||
|
||||
// Erasure code the data into K data blocks and M parity
|
||||
// blocks. Only the parity blocks are filled. Data blocks remain
|
||||
// intact.
|
||||
C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
|
||||
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
|
||||
(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
|
||||
|
||||
return encodedBlocks, nil
|
||||
}
|
||||
82
pkg/erasure/erasure_test.go
Normal file
82
pkg/erasure/erasure_test.go
Normal file
@ -0,0 +1,82 @@
|
||||
// +build !windows
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
const (
|
||||
k = 10
|
||||
m = 5
|
||||
)
|
||||
|
||||
func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
|
||||
for _, err := range errorIndex {
|
||||
chunks[err] = nil
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
func (s *MySuite) TestEncodeDecodeFailure(c *C) {
|
||||
ep, err := ValidateParams(k, m)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
|
||||
|
||||
e := NewErasure(ep)
|
||||
chunks, err := e.Encode(data)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
errorIndex := []int{0, 3, 5, 9, 11, 13}
|
||||
chunks = corruptChunks(chunks, errorIndex)
|
||||
|
||||
_, err = e.Decode(chunks, len(data))
|
||||
c.Assert(err, Not(IsNil))
|
||||
}
|
||||
|
||||
func (s *MySuite) TestEncodeDecodeSuccess(c *C) {
|
||||
ep, err := ValidateParams(k, m)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
|
||||
|
||||
e := NewErasure(ep)
|
||||
chunks, err := e.Encode(data)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
errorIndex := []int{0, 3, 5, 9, 13}
|
||||
chunks = corruptChunks(chunks, errorIndex)
|
||||
|
||||
recoveredData, err := e.Decode(chunks, len(data))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
if !bytes.Equal(data, recoveredData) {
|
||||
c.Fatalf("Recovered data mismatches with original data")
|
||||
}
|
||||
}
|
||||
38
pkg/erasure/stdint.go
Normal file
38
pkg/erasure/stdint.go
Normal file
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package erasure
|
||||
|
||||
//
|
||||
// int sizeInt()
|
||||
// {
|
||||
// return sizeof(int);
|
||||
// }
|
||||
import "C"
|
||||
import "unsafe"
|
||||
|
||||
var (
|
||||
// See http://golang.org/ref/spec#Numeric_types
|
||||
sizeInt = int(C.sizeInt())
|
||||
// SizeInt8 is the byte size of a int8.
|
||||
sizeInt8 = int(unsafe.Sizeof(int8(0)))
|
||||
// SizeInt16 is the byte size of a int16.
|
||||
sizeInt16 = int(unsafe.Sizeof(int16(0)))
|
||||
// SizeInt32 is the byte size of a int32.
|
||||
sizeInt32 = int(unsafe.Sizeof(int32(0)))
|
||||
// SizeInt64 is the byte size of a int64.
|
||||
sizeInt64 = int(unsafe.Sizeof(int64(0)))
|
||||
)
|
||||
@ -17,8 +17,8 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio-xl/pkg/quick"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
|
||||
var multipartsMetadataPath, bucketsMetadataPath string
|
||||
|
||||
@ -39,16 +39,6 @@ func (e ExpiredPresignedRequest) Error() string {
|
||||
return "Presigned request already expired"
|
||||
}
|
||||
|
||||
// SignatureDoesNotMatch invalid signature
|
||||
type SignatureDoesNotMatch struct {
|
||||
SignatureSent string
|
||||
SignatureCalculated string
|
||||
}
|
||||
|
||||
func (e SignatureDoesNotMatch) Error() string {
|
||||
return "The request signature we calculated does not match the signature you provided"
|
||||
}
|
||||
|
||||
// InvalidArgument invalid argument
|
||||
type InvalidArgument struct{}
|
||||
|
||||
|
||||
@ -25,8 +25,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/ioutils"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// listObjectsParams - list objects input parameters.
|
||||
|
||||
@ -22,8 +22,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
/// Bucket Operations
|
||||
|
||||
@ -33,12 +33,13 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/atomic"
|
||||
"github.com/minio/minio-xl/pkg/crypto/sha256"
|
||||
"github.com/minio/minio-xl/pkg/crypto/sha512"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/atomic"
|
||||
"github.com/minio/minio/pkg/crypto/sha256"
|
||||
"github.com/minio/minio/pkg/crypto/sha512"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
signV4 "github.com/minio/minio/pkg/signature"
|
||||
)
|
||||
|
||||
// isValidUploadID - is upload id.
|
||||
@ -264,7 +265,7 @@ func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
|
||||
|
||||
// CreateObjectPart - create a part in a multipart session
|
||||
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *Signature) (string, *probe.Error) {
|
||||
func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *signV4.Signature) (string, *probe.Error) {
|
||||
di, err := disk.GetInfo(fs.path)
|
||||
if err != nil {
|
||||
return "", probe.NewError(err)
|
||||
@ -350,7 +351,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
|
||||
}
|
||||
if !ok {
|
||||
partFile.CloseAndPurge()
|
||||
return "", probe.NewError(SignatureDoesNotMatch{})
|
||||
return "", probe.NewError(signV4.SigDoesNotMatch{})
|
||||
}
|
||||
}
|
||||
partFile.Close()
|
||||
@ -398,7 +399,7 @@ func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum s
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - complete a multipart upload and persist the data
|
||||
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
|
||||
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
|
||||
// Check bucket name is valid.
|
||||
if !IsValidBucketName(bucket) {
|
||||
return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
|
||||
@ -447,7 +448,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
|
||||
}
|
||||
if !ok {
|
||||
file.CloseAndPurge()
|
||||
return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
|
||||
return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
|
||||
}
|
||||
}
|
||||
completeMultipartUpload := &CompleteMultipartUpload{}
|
||||
|
||||
@ -28,12 +28,13 @@ import (
|
||||
"encoding/hex"
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/atomic"
|
||||
"github.com/minio/minio-xl/pkg/crypto/sha256"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/atomic"
|
||||
"github.com/minio/minio/pkg/crypto/sha256"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/ioutils"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
signV4 "github.com/minio/minio/pkg/signature"
|
||||
)
|
||||
|
||||
/// Object Operations
|
||||
@ -198,7 +199,7 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
|
||||
}
|
||||
|
||||
// CreateObject - create an object.
|
||||
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
|
||||
func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, signature *signV4.Signature) (ObjectMetadata, *probe.Error) {
|
||||
di, e := disk.GetInfo(fs.path)
|
||||
if e != nil {
|
||||
return ObjectMetadata{}, probe.NewError(e)
|
||||
@ -293,7 +294,7 @@ func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size in
|
||||
}
|
||||
if !ok {
|
||||
file.CloseAndPurge()
|
||||
return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
|
||||
return ObjectMetadata{}, probe.NewError(signV4.SigDoesNotMatch{})
|
||||
}
|
||||
}
|
||||
file.Close()
|
||||
|
||||
@ -22,7 +22,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// Filesystem - local variables
|
||||
|
||||
202
pkg/minhttp/LICENSE.Minio
Normal file
202
pkg/minhttp/LICENSE.Minio
Normal file
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@ -34,7 +34,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/facebookgo/httpdown"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// An app contains one or more servers and their associated configuration.
|
||||
@ -34,7 +34,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/facebookgo/httpdown"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// An app contains one or more servers and their associated configuration.
|
||||
@ -1,4 +1,4 @@
|
||||
// !build windows
|
||||
// +build windows
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
@ -25,7 +25,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// This package is a fork https://github.com/facebookgo/grace
|
||||
@ -34,7 +34,7 @@ import (
|
||||
|
||||
// This package provides a family of Listen functions that either open a
|
||||
// fresh connection or provide an inherited connection from when the process
|
||||
// was started. This behaves like their counterparts in the net pacakge, but
|
||||
// was started. This behaves like their counterparts in the net package, but
|
||||
// transparently provide support for graceful restarts without dropping
|
||||
// connections. This is provided in a systemd socket activation compatible form
|
||||
// to allow using socket activation.
|
||||
@ -19,7 +19,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
@ -44,7 +44,7 @@ func testDummy2() *probe.Error {
|
||||
|
||||
func (s *MySuite) TestProbe(c *C) {
|
||||
probe.Init() // Set project's root source path.
|
||||
probe.SetAppInfo("Release-Tag", "RELEASE.Sat-19-Sep-2015-06-15-16-GMT")
|
||||
probe.SetAppInfo("Commit-ID", "7390cc957239")
|
||||
es := testDummy2().Trace("TopOfStack")
|
||||
// Uncomment the following Println to visually test probe call trace.
|
||||
// fmt.Println("Expecting a simulated error here.", es)
|
||||
@ -30,8 +30,8 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/fatih/structs"
|
||||
"github.com/minio/minio-xl/pkg/atomic"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/atomic"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// Config - generic config interface functions
|
||||
@ -51,7 +51,8 @@ type config struct {
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
|
||||
// CheckData - checks the validity of config data. Data sould be of type struct and contain a string type field called "Version"
|
||||
// CheckData - checks the validity of config data. Data should be of
|
||||
// type struct and contain a string type field called "Version".
|
||||
func CheckData(data interface{}) *probe.Error {
|
||||
if !structs.IsStruct(data) {
|
||||
return probe.NewError(fmt.Errorf("Invalid argument type. Expecing \"struct\" type."))
|
||||
@ -22,7 +22,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/quick"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
48
pkg/signature/errors.go
Normal file
48
pkg/signature/errors.go
Normal file
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package signature
|
||||
|
||||
// MissingDateHeader date header missing
|
||||
type MissingDateHeader struct{}
|
||||
|
||||
func (e MissingDateHeader) Error() string {
|
||||
return "Missing date header"
|
||||
}
|
||||
|
||||
// MissingExpiresQuery expires query string missing
|
||||
type MissingExpiresQuery struct{}
|
||||
|
||||
func (e MissingExpiresQuery) Error() string {
|
||||
return "Missing expires query string"
|
||||
}
|
||||
|
||||
// ExpiredPresignedRequest request already expired
|
||||
type ExpiredPresignedRequest struct{}
|
||||
|
||||
func (e ExpiredPresignedRequest) Error() string {
|
||||
return "Presigned request already expired"
|
||||
}
|
||||
|
||||
// SigDoesNotMatch invalid signature
|
||||
type SigDoesNotMatch struct {
|
||||
SignatureSent string
|
||||
SignatureCalculated string
|
||||
}
|
||||
|
||||
func (e SigDoesNotMatch) Error() string {
|
||||
return "The request signature we calculated does not match the signature you provided"
|
||||
}
|
||||
@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fs
|
||||
package signature
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@ -22,7 +22,7 @@ import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// toString - Safely convert interface to string without causing panic.
|
||||
@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package fs
|
||||
package signature
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -29,8 +29,8 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/minio/minio-xl/pkg/crypto/sha256"
|
||||
"github.com/minio/minio-xl/pkg/probe"
|
||||
"github.com/minio/minio/pkg/crypto/sha256"
|
||||
"github.com/minio/minio/pkg/probe"
|
||||
)
|
||||
|
||||
// Signature - local variables
|
||||
@ -243,7 +243,8 @@ func (r Signature) getScope(t time.Time) string {
|
||||
func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string {
|
||||
stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
|
||||
stringToSign = stringToSign + r.getScope(t) + "\n"
|
||||
stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
|
||||
canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest))
|
||||
stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:])
|
||||
return stringToSign
|
||||
}
|
||||
|
||||
44
pkg/tasker/commands.go
Normal file
44
pkg/tasker/commands.go
Normal file
@ -0,0 +1,44 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package tasker
|
||||
|
||||
// Command is number that uniquely identifies a command function.
|
||||
type Command uint8
|
||||
|
||||
// Enumerate the task commands.
|
||||
const (
|
||||
// CmdNOOP does nothing. It is a default placeholder. Uninitialized variable of this type will point to NOOP command by default.
|
||||
CmdNOOP Command = iota
|
||||
// CmdSignalEnd gracefully ends current task. Never ending tasks (loop over) or Batched jobs will not take the next iteration,
|
||||
// but may finish the current state to completion.
|
||||
CmdSignalEnd
|
||||
// CmdSignalAbort ends the current task at hand immediately. It may still cleanup dangling issues quickly.
|
||||
CmdSignalAbort
|
||||
// CmdSignalSuspend suspends the current task.
|
||||
CmdSignalSuspend
|
||||
// CmdSignalResume resumes a suspended task.
|
||||
CmdSignalResume
|
||||
// CmdPriorityLow is optimized to conserve resources and complete the task at a slow pace. This option is ideal for batch processed tasks.
|
||||
CmdPriorityLow
|
||||
// CmdPriorityMedium is the default priority. It is a balanced option between resources and speed.
|
||||
CmdPriorityMedium
|
||||
// CmdPriorityHigh is optimized for speed. This option is ideal for short lived tasks (like meta-data related) that are latency sensitive. Use this option wisely.
|
||||
CmdPriorityHigh
|
||||
// CmdPrioritySuper is an exclusive priority. All tasks with priority lower than Super (including High) are paused
|
||||
// temporarily until this task completes. Anytime you consider using this priority level, please seek for approval.
|
||||
CmdPrioritySuper
|
||||
)
|
||||
56
pkg/tasker/handle.go
Normal file
56
pkg/tasker/handle.go
Normal file
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package tasker
|
||||
|
||||
import "github.com/minio/minio/pkg/probe"
|
||||
|
||||
// Handle as the name suggests is a handle (self reference) to its
|
||||
// own task structure. Task has limited privileges over itself. Only the
|
||||
// task controller (TaskCtl) can manage the task by sending commands to
|
||||
// the task over channels.
|
||||
type Handle struct {
|
||||
this taskRef
|
||||
cmdCh <-chan Command // Channel to receive commands from TaskCtl.
|
||||
statusCh chan<- status // Channel to send completion status and error (if any) to TaskCtl.
|
||||
closeCh chan<- taskRef // Channel to notify the TaskCtl about ending this task.
|
||||
}
|
||||
|
||||
// Listen returns a channel to receive commands.
|
||||
func (t Handle) Listen() <-chan Command {
|
||||
return t.cmdCh
|
||||
}
|
||||
|
||||
// StatusDone acknowledges successful completion of a command.
|
||||
func (t Handle) StatusDone() {
|
||||
t.statusCh <- status{code: statusDone, err: nil}
|
||||
}
|
||||
|
||||
// StatusBusy rejects a command with busy status.
|
||||
func (t Handle) StatusBusy() {
|
||||
t.statusCh <- status{code: statusBusy, err: nil}
|
||||
}
|
||||
|
||||
// StatusFail returns failure status.
|
||||
func (t Handle) StatusFail(err *probe.Error) {
|
||||
t.statusCh <- status{code: statusFail, err: err}
|
||||
}
|
||||
|
||||
// Close notifies the TaskCtl about the end of this Task. Owner of the
|
||||
// task must invoke Close() when it is done performing its job.
|
||||
func (t Handle) Close() {
|
||||
t.closeCh <- t.this
|
||||
}
|
||||
35
pkg/tasker/status.go
Normal file
35
pkg/tasker/status.go
Normal file
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package tasker
|
||||
|
||||
import "github.com/minio/minio/pkg/probe"
|
||||
|
||||
// StatusCode denotes the completion status of a command.
|
||||
type statusCode int8
|
||||
|
||||
// Enumerate task return status codes.
|
||||
const (
|
||||
statusDone statusCode = iota
|
||||
statusBusy
|
||||
statusFail
|
||||
)
|
||||
|
||||
// Status returns the completion status and error (if any) of a command.
|
||||
type status struct {
|
||||
code statusCode // Completion code.
|
||||
err *probe.Error // Error if any.
|
||||
}
|
||||
103
pkg/tasker/task.go
Normal file
103
pkg/tasker/task.go
Normal file
@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package tasker
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// NOTE: Task is a private entity. It is created and managed by TaskCtl
|
||||
// entirely. Only TaskCtl and Handle objects are exposed outside.
|
||||
|
||||
// taskRef is a unique reference ID to a task. It is assigned by the
|
||||
// TaskCtl during the creation of a task. All tasfRef variables are
|
||||
// named "this".
|
||||
type taskRef *list.Element
|
||||
|
||||
// Task is an abstract concept built on top of Go routines and
|
||||
// channels. Tasks themselves are expected to co-operate and comply with
|
||||
// the TaskCtl commands.
|
||||
|
||||
type task struct {
|
||||
mutex *sync.Mutex
|
||||
|
||||
this taskRef // Refence to task entry in the TaskCtl's task list.
|
||||
name string // Free form name.
|
||||
priority Command // Current priority.
|
||||
cmdCh chan Command // Channel to receive commands from TaskCtl.
|
||||
statusCh chan status // Channel to send completion status and error (if any) to TaskCtl.
|
||||
closeCh chan taskRef // Channel to notify the TaskCtl about ending this task.
|
||||
}
|
||||
|
||||
// NewTask creates a new task structure and returns a handle to
|
||||
// it. Only the task controller has access to the task structure. The
|
||||
// caller routine only receives a handle to its task structure. Task
|
||||
// handle is like a reference to task self. Caller is expected to listen
|
||||
// for commands from the task controller and comply with it co-operatively.
|
||||
// this: Task reference is unique identifier assigned by the TaskCtl.
|
||||
// name: Free form name of the task. Eg. "Late Night Disk Scrubber".
|
||||
func newTask(name string) task {
|
||||
return task{
|
||||
// this: Is set by the TaskCtl's NewTask function.
|
||||
mutex: &sync.Mutex{},
|
||||
name: name,
|
||||
priority: CmdPriorityMedium,
|
||||
cmdCh: make(chan Command),
|
||||
statusCh: make(chan status),
|
||||
closeCh: make(chan taskRef),
|
||||
}
|
||||
}
|
||||
|
||||
// getHandle returns a handle to the task. Handle has limited access to the task structure and it is safe to be exposed.
|
||||
func (t task) getHandle() Handle {
|
||||
t.mutex.Lock()
|
||||
defer t.mutex.Unlock()
|
||||
|
||||
// Make a handle with limited access to channels (only send or receive).
|
||||
return Handle{
|
||||
cmdCh: t.cmdCh,
|
||||
statusCh: t.statusCh,
|
||||
closeCh: t.closeCh,
|
||||
}
|
||||
}
|
||||
|
||||
// command method sends a command code to the task and returns its completion status.
|
||||
func (t task) command(cmd Command) status {
|
||||
t.mutex.Lock()
|
||||
defer t.mutex.Unlock()
|
||||
|
||||
t.cmdCh <- cmd
|
||||
return <-t.statusCh
|
||||
}
|
||||
|
||||
// close releases all the resources held by this task.
|
||||
func (t task) close() {
|
||||
t.mutex.Lock()
|
||||
defer t.mutex.Unlock()
|
||||
|
||||
// Task can be ended in 2 ways.
|
||||
// 1) Calling application invokes Handle.Close().
|
||||
// 2) TaskCtl.Shutdown() ending the task's life.
|
||||
// In either case, task.close() is invoked only via the
|
||||
// TaskCtl. Handle.Close() only sends a message to the TaskCtl to
|
||||
// initiate a close call.
|
||||
|
||||
close(t.cmdCh)
|
||||
close(t.statusCh)
|
||||
close(t.closeCh)
|
||||
}
|
||||
164
pkg/tasker/taskctl.go
Normal file
164
pkg/tasker/taskctl.go
Normal file
@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package tasker
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// TaskCtl (Task Controller) is a framework to create and manage
|
||||
// tasks.
|
||||
type TaskCtl struct {
|
||||
mutex *sync.Mutex // Lock
|
||||
// List of tasks managed by this task controller.
|
||||
tasks *list.List
|
||||
}
|
||||
|
||||
// New creates a new TaskCtl to create and control a collection of tasks.
|
||||
// Single application can create multiple task controllers to manage different set of tasks separately.
|
||||
func New(name string) *TaskCtl {
|
||||
return &TaskCtl{
|
||||
mutex: &sync.Mutex{},
|
||||
tasks: list.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// NewTask creates a new task structure and returns a handle to it. Only the task controller
|
||||
// has access to the task structure. The caller routine only receives a handle to its task structure.
|
||||
// Task handle is like a reference to task self. Caller is expected to listen for commands from
|
||||
// the task controller and comply with it co-operatively.
|
||||
func (tc *TaskCtl) NewTask(name string) Handle {
|
||||
tc.mutex.Lock()
|
||||
defer tc.mutex.Unlock()
|
||||
|
||||
// Create a new task.
|
||||
tsk := newTask(name)
|
||||
|
||||
// Register this task in the TaskCtl's tasklist and save the reference.
|
||||
tsk.this = tc.tasks.PushBack(tsk)
|
||||
|
||||
// Free task from the tasklist upon close call.
|
||||
go func() {
|
||||
// Release the tasks resources upon return of this function.
|
||||
defer tsk.close()
|
||||
|
||||
// Will be notified here upon task's end of life.
|
||||
this := <-tsk.closeCh
|
||||
|
||||
tc.mutex.Lock()
|
||||
defer tc.mutex.Unlock()
|
||||
|
||||
// Release the task structure from the task list.
|
||||
tc.tasks.Remove(this)
|
||||
}()
|
||||
|
||||
// Return a handle to this task.
|
||||
return tsk.getHandle()
|
||||
}
|
||||
|
||||
// Shutdown ends all tasks, including the suspended ones.
|
||||
func (tc *TaskCtl) Shutdown() {
|
||||
tc.mutex.Lock()
|
||||
defer tc.mutex.Unlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// End all tasks.
|
||||
for e := tc.tasks.Front(); e != nil; e = e.Next() {
|
||||
wg.Add(1)
|
||||
thisTask := e.Value.(task) // Make a local copy for go routine.
|
||||
// End tasks in background. Flow of events from here is as follows: thisTask.handle.Close() -> tc.NewTask() -> this.task.close().
|
||||
go func() {
|
||||
thisTask.getHandle().Close()
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait() // Wait for all tasks to end gracefully.
|
||||
|
||||
// Reset the task pool.
|
||||
tc.tasks = nil
|
||||
}
|
||||
|
||||
// Suspend puts all tasks to sleep.
|
||||
func (tc *TaskCtl) Suspend() bool {
|
||||
tc.mutex.Lock()
|
||||
defer tc.mutex.Unlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// If any one of the task fails to suspend, this flag will set to false.
|
||||
statusAll := make([]status, tc.tasks.Len())
|
||||
|
||||
// Suspend all tasks.
|
||||
i := 0
|
||||
for e := tc.tasks.Front(); e != nil; e = e.Next() {
|
||||
wg.Add(1)
|
||||
locTask := e.Value.(task) // Make a local copy for go routine.
|
||||
locI := i // local i
|
||||
// Suspend a task in background.
|
||||
go func(locI int) {
|
||||
defer wg.Done()
|
||||
statusAll[locI] = locTask.command(CmdSignalSuspend)
|
||||
}(locI)
|
||||
i++
|
||||
}
|
||||
|
||||
wg.Wait() // Wait for all tasks to suspend gracefully.
|
||||
|
||||
for _, st := range statusAll {
|
||||
if st.code != statusDone {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Resume wakes up all suspended task from sleep.
|
||||
func (tc *TaskCtl) Resume() bool {
|
||||
tc.mutex.Lock()
|
||||
defer tc.mutex.Unlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// If any one of the task fails to suspend, this flag will set to false.
|
||||
statusAll := make([]status, tc.tasks.Len())
|
||||
|
||||
i := 0
|
||||
// Resume all suspended tasks.
|
||||
for e := tc.tasks.Front(); e != nil; e = e.Next() {
|
||||
wg.Add(1)
|
||||
locTask := e.Value.(task) // Make a local copy for go routine.
|
||||
locI := i // local i
|
||||
// Resume a task in background.
|
||||
go func(locI int) {
|
||||
defer wg.Done()
|
||||
statusAll[locI] = locTask.command(CmdSignalResume)
|
||||
}(locI)
|
||||
i++
|
||||
}
|
||||
wg.Wait() // Wait for all tasks to resume.
|
||||
|
||||
for _, st := range statusAll {
|
||||
if st.code != statusDone {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
38
pkg/tasker/taskctl_test.go
Normal file
38
pkg/tasker/taskctl_test.go
Normal file
@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Quick - Quick key value store for config files and persistent state files
|
||||
*
|
||||
* Minio Client (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package tasker_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/tasker"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
|
||||
type MySuite struct{}
|
||||
|
||||
var _ = Suite(&MySuite{})
|
||||
|
||||
func (s *MySuite) TestCheckData(c *C) {
|
||||
testTasks := tasker.New("Test Task")
|
||||
testTasks.Shutdown()
|
||||
// c.Assert(err, Not(IsNil))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user