listObjects: list objects minor optimization.

Minor optimization.

- Add 1000 entries buffered channel for walkerCh.
- Reset marker after the lexical order has reached and
  compare only if the marker is set.
This commit is contained in:
Harshavardhana 2016-02-09 21:38:50 -08:00
parent 4ef01bc225
commit 9b29af8bbe
4 changed files with 14 additions and 8 deletions

View File

@ -24,7 +24,7 @@ build_script:
- curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS% - curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS%
- curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR% - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR%
- gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728 - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
- gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS% - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%
- go test . - go test .
- go test -race . - go test -race .
- go test github.com/minio/minio/pkg... - go test github.com/minio/minio/pkg...

View File

@ -176,7 +176,7 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
// Resource handler ServeHTTP() wrapper // Resource handler ServeHTTP() wrapper
func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Skip the first element which is usally '/' and split the rest. // Skip the first element which is usually '/' and split the rest.
splits := strings.SplitN(r.URL.Path[1:], "/", 2) splits := strings.SplitN(r.URL.Path[1:], "/", 2)
// Save bucketName and objectName extracted from url Path. // Save bucketName and objectName extracted from url Path.

View File

@ -56,11 +56,11 @@ type listWorkerReq struct {
respCh chan ListObjectsResult respCh chan ListObjectsResult
} }
// listObjects - list objects lists objects upto maxKeys for a given prefix. // listObjects - list objects lists objects up to maxKeys for a given prefix.
func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) { func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) {
quitWalker := make(chan bool) quitWalker := make(chan bool)
reqCh := make(chan listWorkerReq) reqCh := make(chan listWorkerReq)
walkerCh := make(chan ObjectMetadata) walkerCh := make(chan ObjectMetadata, 1000)
go func() { go func() {
defer close(walkerCh) defer close(walkerCh)
var walkPath string var walkPath string
@ -144,8 +144,14 @@ func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKe
for object := range walkerCh { for object := range walkerCh {
// Verify if the object is lexically smaller than // Verify if the object is lexically smaller than
// the marker, we will skip those objects. // the marker, we will skip those objects.
if marker >= object.Object { if marker != "" {
continue if marker >= object.Object {
continue
} else {
// Reset marker so that we avoid comparing
// again and again in a loop unecessarily.
marker = ""
}
} }
if delimiter != "" { if delimiter != "" {
// Prefixes are only valid wth delimiters, and // Prefixes are only valid wth delimiters, and
@ -275,7 +281,7 @@ func (fs *Filesystem) listObjectsService() *probe.Error {
return nil return nil
} }
// ListObjects - lists all objects for a given prefix, returns upto // ListObjects - lists all objects for a given prefix, returns up to
// maxKeys number of objects per call. // maxKeys number of objects per call.
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) { func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
// Input validation. // Input validation.

View File

@ -346,7 +346,7 @@ func (r *Signature) DoesPresignedSignatureMatch() (bool, *probe.Error) {
// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html // - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
// returns true if matches, false otherwise. if error is not nil then it is always false // returns true if matches, false otherwise. if error is not nil then it is always false
func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
// set new calulated payload // set new calculated payload
r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload) r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload)
// Add date if not present throw error // Add date if not present throw error