MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-01-24 13:13:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1113 from harshavardhana/list-objects-optimization

Browse Source 
listObjects: list objects minor optimization.
This commit is contained in:
Harshavardhana 2016-02-10 10:33:37 -08:00
commit 31dbdb1787
6 changed files with 30 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Makefile
View File

@ -26,6 +26,7 @@ getdeps: checkdeps checkgopath
@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:" @go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:" @go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:" @go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
$(UI_ASSETS): $(UI_ASSETS):
@curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:" @curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:"
@ -33,7 +34,7 @@ $(UI_ASSETS):
@curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:" @curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:"
@gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets." @gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets."
verifiers: getdeps vet fmt lint cyclo verifiers: getdeps vet fmt lint cyclo spelling
vet: vet:
@echo "Running $@:" @echo "Running $@:"
@ -61,7 +62,11 @@ build: getdeps verifiers $(UI_ASSETS)
@echo "Installing minio:" @echo "Installing minio:"
deadcode: deadcode:
@GO15VENDOREXPERIMENT=1 deadcode @GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
spelling:
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell *.go
@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
test: build test: build
@echo "Running all testing:" @echo "Running all testing:"
@ -72,13 +77,13 @@ gomake-all: build
@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio @GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
pkg-add: pkg-add:
@GO15VENDOREXPERIMENT=1 govendor add $(PKG) @GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
pkg-update: pkg-update:
@GO15VENDOREXPERIMENT=1 govendor update $(PKG) @GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor update $(PKG)
pkg-remove: pkg-remove:
@GO15VENDOREXPERIMENT=1 govendor remove $(PKG) @GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor remove $(PKG)
install: gomake-all install: gomake-all

6
buildscripts/checkdeps.sh
View File

@ -21,6 +21,7 @@ _init() {
## Minimum required versions for build dependencies ## Minimum required versions for build dependencies
GIT_VERSION="1.0" GIT_VERSION="1.0"
CURL_VERSION="7.12.0"
GPG_VERSION="1.3" GPG_VERSION="1.3"
GO_VERSION="1.5.1" GO_VERSION="1.5.1"
OSX_VERSION="10.8" OSX_VERSION="10.8"
@ -186,6 +187,11 @@ check_deps() {
if [ $? -ge 2 ]; then if [ $? -ge 2 ]; then
MISSING="${MISSING} gpg" MISSING="${MISSING} gpg"
fi fi
check_version "$(env curl --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g' | head -1)" "${CURL_VERSION}"
if [ $? -ge 2 ]; then
MISSING="${MISSING} curl"
fi
} }
main() { main() {

2
generic-handlers.go
View File

@ -176,7 +176,7 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
// Resource handler ServeHTTP() wrapper // Resource handler ServeHTTP() wrapper
func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Skip the first element which is usally '/' and split the rest. // Skip the first element which is usually '/' and split the rest.
splits := strings.SplitN(r.URL.Path[1:], "/", 2) splits := strings.SplitN(r.URL.Path[1:], "/", 2)
// Save bucketName and objectName extracted from url Path. // Save bucketName and objectName extracted from url Path.

16
pkg/fs/fs-bucket-listobjects.go
View File

@ -56,11 +56,11 @@ type listWorkerReq struct {
respCh chan ListObjectsResult respCh chan ListObjectsResult
} }
// listObjects - list objects lists objects upto maxKeys for a given prefix. // listObjects - list objects lists objects up to maxKeys for a given prefix.
func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) { func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) {
quitWalker := make(chan bool) quitWalker := make(chan bool)
reqCh := make(chan listWorkerReq) reqCh := make(chan listWorkerReq)
walkerCh := make(chan ObjectMetadata) walkerCh := make(chan ObjectMetadata, 1000)
go func() { go func() {
defer close(walkerCh) defer close(walkerCh)
var walkPath string var walkPath string
@ -144,8 +144,14 @@ func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKe
for object := range walkerCh { for object := range walkerCh {
// Verify if the object is lexically smaller than // Verify if the object is lexically smaller than
// the marker, we will skip those objects. // the marker, we will skip those objects.
if marker >= object.Object { if marker != "" {
continue if marker >= object.Object {
continue
} else {
// Reset marker so that we avoid comparing
// again and again in a loop unecessarily.
marker = ""
}
} }
if delimiter != "" { if delimiter != "" {
// Prefixes are only valid wth delimiters, and // Prefixes are only valid wth delimiters, and
@ -275,7 +281,7 @@ func (fs *Filesystem) listObjectsService() *probe.Error {
return nil return nil
} }
// ListObjects - lists all objects for a given prefix, returns upto // ListObjects - lists all objects for a given prefix, returns up to
// maxKeys number of objects per call. // maxKeys number of objects per call.
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) { func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
// Input validation. // Input validation.

2
pkg/fs/signature.go
View File

@ -346,7 +346,7 @@ func (r *Signature) DoesPresignedSignatureMatch() (bool, *probe.Error) {
// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html // - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
// returns true if matches, false otherwise. if error is not nil then it is always false // returns true if matches, false otherwise. if error is not nil then it is always false
func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
// set new calulated payload // set new calculated payload
r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload) r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload)
// Add date if not present throw error // Add date if not present throw error