mirror of https://github.com/minio/minio.git
Adding lru
This commit is contained in:
parent
b6daf99dcd
commit
5fc5f79ab7
|
@ -0,0 +1,150 @@
|
|||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
---
|
||||
Modifications from Minio under the following license:
|
||||
|
||||
Minimalist Object Storage, (C) 2015 Minio, Inc.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package lru implements an LRU cache.
|
||||
package lru
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
)
|
||||
|
||||
// Cache is an LRU cache. It is not safe for concurrent access.
|
||||
type Cache struct {
|
||||
// MaxEntries is the maximum number of cache entries before
|
||||
// an item is evicted. Zero means no limit.
|
||||
MaxEntries int
|
||||
|
||||
// OnEvicted optionally specificies a callback function to be
|
||||
// executed when an entry is purged from the cache.
|
||||
OnEvicted func(key Key, value interface{})
|
||||
|
||||
ll *list.List
|
||||
cache map[interface{}]*list.Element
|
||||
}
|
||||
|
||||
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
|
||||
type Key interface{}
|
||||
|
||||
type entry struct {
|
||||
key Key
|
||||
value interface{}
|
||||
}
|
||||
|
||||
// New creates a new Cache.
|
||||
// If maxEntries is zero, the cache has no limit and it's assumed
|
||||
// that eviction is done by the caller.
|
||||
func New(maxEntries int) *Cache {
|
||||
return &Cache{
|
||||
MaxEntries: maxEntries,
|
||||
ll: list.New(),
|
||||
cache: make(map[interface{}]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds a value to the cache.
|
||||
func (c *Cache) Add(key Key, value interface{}) {
|
||||
if c.cache == nil {
|
||||
c.cache = make(map[interface{}]*list.Element)
|
||||
c.ll = list.New()
|
||||
}
|
||||
if ee, ok := c.cache[key]; ok {
|
||||
c.ll.MoveToFront(ee)
|
||||
ee.Value.(*entry).value = value
|
||||
return
|
||||
}
|
||||
ele := c.ll.PushFront(&entry{key, value})
|
||||
c.cache[key] = ele
|
||||
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
|
||||
c.RemoveOldest()
|
||||
}
|
||||
}
|
||||
|
||||
// Get looks up a key's value from the cache.
|
||||
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
|
||||
if c.cache == nil {
|
||||
return
|
||||
}
|
||||
if ele, hit := c.cache[key]; hit {
|
||||
c.ll.MoveToFront(ele)
|
||||
return ele.Value.(*entry).value, true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Remove removes the provided key from the cache.
|
||||
func (c *Cache) Remove(key Key) {
|
||||
if c.cache == nil {
|
||||
return
|
||||
}
|
||||
if ele, hit := c.cache[key]; hit {
|
||||
c.removeElement(ele)
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveOldest removes the oldest item from the cache.
|
||||
func (c *Cache) RemoveOldest() {
|
||||
if c.cache == nil {
|
||||
return
|
||||
}
|
||||
ele := c.ll.Back()
|
||||
if ele != nil {
|
||||
c.removeElement(ele)
|
||||
}
|
||||
}
|
||||
|
||||
// GetOldest returns the oldest key, value, ok without modifying the lru
|
||||
func (c *Cache) GetOldest() (key Key, value interface{}, ok bool) {
|
||||
if c.cache == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
ele := c.ll.Back()
|
||||
if ele != nil {
|
||||
return ele.Value.(*entry).key, ele.Value.(*entry).value, true
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
func (c *Cache) removeElement(e *list.Element) {
|
||||
c.ll.Remove(e)
|
||||
kv := e.Value.(*entry)
|
||||
delete(c.cache, kv.key)
|
||||
if c.OnEvicted != nil {
|
||||
c.OnEvicted(kv.key, kv.value)
|
||||
}
|
||||
}
|
||||
|
||||
// Len returns the number of items in the cache.
|
||||
func (c *Cache) Len() int {
|
||||
if c.cache == nil {
|
||||
return 0
|
||||
}
|
||||
return c.ll.Len()
|
||||
}
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
---
|
||||
Modifications from Minio under the following license:
|
||||
|
||||
Minimalist Object Storage, (C) 2015 Minio, Inc.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package lru
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
type simpleStruct struct {
|
||||
int
|
||||
string
|
||||
}
|
||||
|
||||
type complexStruct struct {
|
||||
int
|
||||
simpleStruct
|
||||
}
|
||||
|
||||
var getTests = []struct {
|
||||
name string
|
||||
keyToAdd interface{}
|
||||
keyToGet interface{}
|
||||
expectedOk bool
|
||||
}{
|
||||
{"string_hit", "myKey", "myKey", true},
|
||||
{"string_miss", "myKey", "nonsense", false},
|
||||
{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
|
||||
{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
|
||||
{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
|
||||
complexStruct{1, simpleStruct{2, "three"}}, true},
|
||||
}
|
||||
|
||||
func TestGet(t *testing.T) {
|
||||
for _, tt := range getTests {
|
||||
lru := New(0)
|
||||
lru.Add(tt.keyToAdd, 1234)
|
||||
val, ok := lru.Get(tt.keyToGet)
|
||||
if ok != tt.expectedOk {
|
||||
t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
|
||||
} else if ok && val != 1234 {
|
||||
t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemove(t *testing.T) {
|
||||
lru := New(0)
|
||||
lru.Add("myKey", 1234)
|
||||
if val, ok := lru.Get("myKey"); !ok {
|
||||
t.Fatal("TestRemove returned no match")
|
||||
} else if val != 1234 {
|
||||
t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val)
|
||||
}
|
||||
|
||||
lru.Remove("myKey")
|
||||
if _, ok := lru.Get("myKey"); ok {
|
||||
t.Fatal("TestRemove returned a removed entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldest(t *testing.T) {
|
||||
lru := New(0)
|
||||
lru.Add("a", 1)
|
||||
lru.Add("b", 2)
|
||||
lru.Add("c", 3)
|
||||
key, val, ok := lru.GetOldest()
|
||||
if ok != true {
|
||||
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
|
||||
}
|
||||
if key != "a" && val != 1 {
|
||||
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
|
||||
}
|
||||
lru.RemoveOldest()
|
||||
key, val, ok = lru.GetOldest()
|
||||
if ok != true {
|
||||
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
|
||||
}
|
||||
if key != "b" && val != 2 {
|
||||
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
|
||||
}
|
||||
}
|
|
@ -33,9 +33,9 @@ import (
|
|||
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/golang/groupcache/lru"
|
||||
"github.com/minio-io/minio/pkg/iodine"
|
||||
"github.com/minio-io/minio/pkg/storage/drivers"
|
||||
"github.com/minio-io/minio/pkg/storage/drivers/memory/lru"
|
||||
"github.com/minio-io/minio/pkg/utils/log"
|
||||
"github.com/minio-io/minio/pkg/utils/split"
|
||||
)
|
||||
|
@ -60,6 +60,7 @@ type storedBucket struct {
|
|||
|
||||
type storedObject struct {
|
||||
metadata drivers.ObjectMetadata
|
||||
lastAccessed time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
|
@ -483,29 +484,20 @@ func (memory *memoryDriver) expireObjects() {
|
|||
if memory.shutdown {
|
||||
return
|
||||
}
|
||||
var keysToRemove []string
|
||||
if len(memory.objectMetadata) > 0 {
|
||||
memory.lock.RLock()
|
||||
var earliest time.Time
|
||||
for key, object := range memory.objectMetadata {
|
||||
if time.Now().Add(-memory.expiration).After(object.metadata.Created) {
|
||||
keysToRemove = append(keysToRemove, key)
|
||||
} else {
|
||||
if object.metadata.Created.Before(earliest) {
|
||||
earliest = object.metadata.Created
|
||||
}
|
||||
}
|
||||
}
|
||||
memory.lock.RUnlock()
|
||||
var sleepDuration time.Duration
|
||||
memory.lock.Lock()
|
||||
for _, key := range keysToRemove {
|
||||
memory.objects.Remove(key)
|
||||
if k, _, ok := memory.objects.GetOldest(); ok {
|
||||
key := k.(string)
|
||||
object := memory.objectMetadata[key]
|
||||
if time.Now().Sub(object.lastAccessed) > memory.expiration {
|
||||
memory.objects.RemoveOldest()
|
||||
} else {
|
||||
sleepDuration = memory.expiration - time.Now().Sub(object.lastAccessed)
|
||||
}
|
||||
}
|
||||
memory.lock.Unlock()
|
||||
sleepFor := earliest.Sub(time.Now())
|
||||
if sleepFor > 0 {
|
||||
time.Sleep(sleepFor)
|
||||
}
|
||||
time.Sleep(sleepDuration)
|
||||
} else {
|
||||
time.Sleep(memory.expiration)
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue