MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-20 01:50:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite cache implementation to cache only on GET (#7694)

Browse Source 
Fixes #7458
Fixes #7573 
Fixes #7938 
Fixes #6934
Fixes #6265 
Fixes #6630 

This will allow the cache to consistently work for
server and gateways. Range GET requests will
be cached in the background after the request
is served from the backend.

- All cached content is automatically bitrot protected.

- Avoid ETag verification if a cache-control header
is set and the cached content is still valid.

- This PR changes the cache backend format, and all existing
content will be migrated to the new format. Until the data is
migrated completely, all content will be served from the backend.
This commit is contained in:
poornas
2019-08-09 17:09:08 -07:00
committed by Harshavardhana
parent 1ce8d2c476
commit 3385bf3da8
15 changed files with 1423 additions and 1686 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cmd/web-handlers.go
View File

@@ -257,9 +257,6 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
}
deleteBucket := objectAPI.DeleteBucket
if web.CacheAPI() != nil {
deleteBucket = web.CacheAPI().DeleteBucket
}
if err := deleteBucket(ctx, args.BucketName); err != nil {
return toJSONError(ctx, err, args.BucketName)
@@ -302,9 +299,6 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
return toJSONError(ctx, errServerNotInitialized)
}
listBuckets := objectAPI.ListBuckets
if web.CacheAPI() != nil {
listBuckets = web.CacheAPI().ListBuckets
}
claims, owner, authErr := webRequestAuthenticate(r)
if authErr != nil {
@@ -407,9 +401,6 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
}
listObjects := objectAPI.ListObjects
if web.CacheAPI() != nil {
listObjects = web.CacheAPI().ListObjects
}
if isRemoteCallRequired(ctx, args.BucketName, objectAPI) {
sr, err := globalDNSConfig.Get(args.BucketName)
@@ -600,9 +591,6 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
return toJSONError(ctx, errServerNotInitialized)
}
listObjects := objectAPI.ListObjects
if web.CacheAPI() != nil {
listObjects = web.CacheAPI().ListObjects
}
claims, owner, authErr := webRequestAuthenticate(r)
if authErr != nil {
@@ -1065,10 +1053,8 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
}
putObject := objectAPI.PutObject
if !hasServerSideEncryptionHeader(r.Header) && web.CacheAPI() != nil {
putObject = web.CacheAPI().PutObject
}
objInfo, err := putObject(ctx, bucket, object, pReader, opts)
objInfo, err := putObject(context.Background(), bucket, object, pReader, opts)
if err != nil {
writeWebErrorResponse(w, err)
return
@@ -1307,32 +1293,29 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
writeWebErrorResponse(w, errInvalidBucketName)
return
}
getObjectNInfo := objectAPI.GetObjectNInfo
if web.CacheAPI() != nil {
getObjectNInfo = web.CacheAPI().GetObjectNInfo
}
getObject := objectAPI.GetObject
if web.CacheAPI() != nil {
getObject = web.CacheAPI().GetObject
}
listObjects := objectAPI.ListObjects
if web.CacheAPI() != nil {
listObjects = web.CacheAPI().ListObjects
}
archive := zip.NewWriter(w)
defer archive.Close()
getObjectInfo := objectAPI.GetObjectInfo
if web.CacheAPI() != nil {
getObjectInfo = web.CacheAPI().GetObjectInfo
}
opts := ObjectOptions{}
var length int64
for _, object := range args.Objects {
// Writes compressed object file to the response.
zipit := func(objectName string) error {
info, err := getObjectInfo(ctx, args.BucketName, objectName, opts)
var opts ObjectOptions
gr, err := getObjectNInfo(ctx, args.BucketName, objectName, nil, r.Header, readLock, opts)
if err != nil {
return err
}
defer gr.Close()
info := gr.ObjInfo
length = info.Size
if objectAPI.IsEncryptionSupported() {
if _, err = DecryptObjectInfo(&info, r.Header); err != nil {
@@ -1395,7 +1378,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
// Response writer should be limited early on for decryption upto required length,
// additionally also skipping mod(offset)64KiB boundaries.
writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
writer, startOffset, length, err = DecryptBlocksRequest(writer, r,
writer, _, length, err = DecryptBlocksRequest(writer, r,
args.BucketName, objectName, startOffset, length, info, false)
if err != nil {
writeWebErrorResponse(w, err)
@@ -1403,14 +1386,20 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
}
}
httpWriter := ioutil.WriteOnClose(writer)
if err = getObject(ctx, args.BucketName, objectName, startOffset, length, httpWriter, "", opts); err != nil {
// Write object content to response body
if _, err = io.Copy(httpWriter, gr); err != nil {
httpWriter.Close()
if info.IsCompressed() {
// Wait for decompression go-routine to retire.
wg.Wait()
}
if !httpWriter.HasWritten() { // write error response only if no data or headers has been written to client yet
writeWebErrorResponse(w, err)
}
return err
}
if err = httpWriter.Close(); err != nil {
if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
writeWebErrorResponse(w, err)