mirror of
https://github.com/minio/minio.git
synced 2025-11-09 05:34:56 -05:00
Remove healing of incomplete multipart uploads (#5390)
Since the server performs automatic clean-up of multipart uploads that have not been resumed for more than a couple of weeks, it was decided to remove functionality to heal multipart uploads.
This commit is contained in:
Aditya Manthramurthy
committed by
kannappanr
kannappanr
parent
20584dc08f
commit
cd22feecf8
@@ -43,8 +43,6 @@ func main() {
|
||||
| | |[`HealBucket`](#HealBucket) |||
|
||||
| | |[`HealObject`](#HealObject)|||
|
||||
| | |[`HealFormat`](#HealFormat)|||
|
||||
| | |[`ListUploadsHeal`](#ListUploadsHeal)|||
|
||||
| | |[`HealUpload`](#HealUpload)|||
|
||||
|
||||
## 1. Constructor
|
||||
<a name="Minio"></a>
|
||||
@@ -315,73 +313,6 @@ __Example__
|
||||
log.Println("successfully healed storage format on available disks.")
|
||||
|
||||
```
|
||||
<a name="ListUploadsHeal"> </a>
|
||||
### ListUploadsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan UploadInfo, error)
|
||||
List ongoing multipart uploads that need healing.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
|`ui.Key` | _string_ | Name of the object being uploaded |
|
||||
|`ui.UploadID` | _string_ | UploadID of the ongoing multipart upload |
|
||||
|`ui.HealUploadInfo.Status` | _HealStatus_| One of `Healthy`, `CanHeal`, `Corrupted`, `QuorumUnavailable`|
|
||||
|`ui.Err`| _error_ | non-nil if fetching fetching healing information failed |
|
||||
|
||||
__Example__
|
||||
|
||||
``` go
|
||||
|
||||
// Set true if recursive listing is needed.
|
||||
isRecursive := true
|
||||
// List objects that need healing for a given bucket and
|
||||
// prefix.
|
||||
healUploadsCh, err := madmClnt.ListUploadsHeal(bucket, prefix, isRecursive, doneCh)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to get list of uploads to be healed: ", err)
|
||||
}
|
||||
|
||||
for upload := range healUploadsCh {
|
||||
if upload.Err != nil {
|
||||
log.Println("upload listing error: ", upload.Err)
|
||||
}
|
||||
|
||||
if upload.HealUploadInfo != nil {
|
||||
switch healInfo := *upload.HealUploadInfo; healInfo.Status {
|
||||
case madmin.CanHeal:
|
||||
fmt.Println(upload.Key, " can be healed.")
|
||||
case madmin.QuorumUnavailable:
|
||||
fmt.Println(upload.Key, " can't be healed until quorum is available.")
|
||||
case madmin.Corrupted:
|
||||
fmt.Println(upload.Key, " can't be healed, not enough information.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
<a name="HealUpload"></a>
|
||||
### HealUpload(bucket, object, uploadID string, isDryRun bool) (HealResult, error)
|
||||
If upload is successfully healed returns nil, otherwise returns error indicating the reason for failure. If isDryRun is true, then the upload is not healed, but heal upload request is validated by the server. e.g, if the upload exists, if upload name is valid etc.
|
||||
|
||||
| Param | Type | Description |
|
||||
|---|---|---|
|
||||
|`h.State` | _HealState_ | Represents the result of heal operation. It could be one of `HealNone`, `HealPartial` or `HealOK`. |
|
||||
|
||||
|
||||
| Value | Description |
|
||||
|---|---|
|
||||
| `HealNone` | Object/Upload wasn't healed on any of the disks |
|
||||
| `HealPartial` | Object/Upload was healed on some of the disks needing heal |
|
||||
| `HealOK` | Object/Upload was healed on all the disks needing heal |
|
||||
|
||||
``` go
|
||||
isDryRun = false
|
||||
healResult, err := madmClnt.HealUpload("mybucket", "myobject", "myUploadID", isDryRun)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Println("Heal-upload result: ", healResult)
|
||||
```
|
||||
|
||||
## 6. Config operations
|
||||
|
||||
@@ -459,4 +390,3 @@ __Example__
|
||||
log.Println("New credentials successfully set.")
|
||||
|
||||
```
|
||||
|
||||
|
||||
@@ -29,10 +29,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxUploadsList = 1000
|
||||
)
|
||||
|
||||
// listBucketHealResult container for listObjects response.
|
||||
type listBucketHealResult struct {
|
||||
// A response can contain CommonPrefixes only if you have
|
||||
@@ -168,76 +164,16 @@ type ObjectInfo struct {
|
||||
HealObjectInfo *HealObjectInfo `json:"healObjectInfo,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInfo - represents an ongoing upload that needs to be healed.
|
||||
type UploadInfo struct {
|
||||
Key string `json:"name"` // Name of the object being uploaded.
|
||||
|
||||
UploadID string `json:"uploadId"` // UploadID
|
||||
// Owner name.
|
||||
Owner struct {
|
||||
DisplayName string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
} `json:"owner"`
|
||||
|
||||
// The class of storage used to store the object.
|
||||
StorageClass string `json:"storageClass"`
|
||||
|
||||
Initiated time.Time `json:"initiated"` // Time at which upload was initiated.
|
||||
|
||||
// Error
|
||||
Err error `json:"-"`
|
||||
HealUploadInfo *HealObjectInfo `json:"healObjectInfo,omitempty"`
|
||||
}
|
||||
|
||||
// Initiator - has same properties as Owner.
|
||||
type Initiator Owner
|
||||
|
||||
// upload - represents an ongoing multipart upload.
|
||||
type upload struct {
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
Initiator Initiator
|
||||
Owner Owner
|
||||
StorageClass string
|
||||
Initiated time.Time
|
||||
HealUploadInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"`
|
||||
}
|
||||
|
||||
// listUploadsHealResponse - represents ListUploadsHeal response.
|
||||
type listUploadsHealResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
|
||||
|
||||
Bucket string
|
||||
KeyMarker string
|
||||
UploadIDMarker string `xml:"UploadIdMarker"`
|
||||
NextKeyMarker string
|
||||
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
|
||||
Delimiter string
|
||||
Prefix string
|
||||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
MaxUploads int
|
||||
IsTruncated bool
|
||||
|
||||
// List of pending uploads.
|
||||
Uploads []upload `xml:"Upload"`
|
||||
|
||||
// Delimed common prefixes.
|
||||
CommonPrefixes []commonPrefix
|
||||
}
|
||||
|
||||
type healQueryKey string
|
||||
|
||||
const (
|
||||
healBucket healQueryKey = "bucket"
|
||||
healObject healQueryKey = "object"
|
||||
healPrefix healQueryKey = "prefix"
|
||||
healMarker healQueryKey = "marker"
|
||||
healDelimiter healQueryKey = "delimiter"
|
||||
healMaxKey healQueryKey = "max-key"
|
||||
healDryRun healQueryKey = "dry-run"
|
||||
healUploadIDMarker healQueryKey = "upload-id-marker"
|
||||
healMaxUpload healQueryKey = "max-uploads"
|
||||
healUploadID healQueryKey = "upload-id"
|
||||
healBucket healQueryKey = "bucket"
|
||||
healObject healQueryKey = "object"
|
||||
healPrefix healQueryKey = "prefix"
|
||||
healMarker healQueryKey = "marker"
|
||||
healDelimiter healQueryKey = "delimiter"
|
||||
healMaxKey healQueryKey = "max-key"
|
||||
healDryRun healQueryKey = "dry-run"
|
||||
)
|
||||
|
||||
// mkHealQueryVal - helper function to construct heal REST API query params.
|
||||
@@ -439,59 +375,6 @@ func (adm *AdminClient) HealBucket(bucket string, dryrun bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HealUpload - Heal the given upload.
|
||||
func (adm *AdminClient) HealUpload(bucket, object, uploadID string, dryrun bool) (HealResult, error) {
|
||||
// Construct query params.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(healBucket), bucket)
|
||||
queryVal.Set(string(healObject), object)
|
||||
queryVal.Set(string(healUploadID), uploadID)
|
||||
if dryrun {
|
||||
queryVal.Set(string(healDryRun), "")
|
||||
}
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "upload")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Execute POST on
|
||||
// /?heal&bucket=mybucket&object=myobject&upload-id=uploadID
|
||||
// to heal an upload.
|
||||
resp, err := adm.executeMethod("POST", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return HealResult{}, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
// Healing is not performed so heal object result is empty.
|
||||
if dryrun {
|
||||
return HealResult{}, nil
|
||||
}
|
||||
|
||||
jsonBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
}
|
||||
|
||||
healResult := HealResult{}
|
||||
err = json.Unmarshal(jsonBytes, &healResult)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
}
|
||||
|
||||
return healResult, nil
|
||||
}
|
||||
|
||||
// HealResult - represents result of heal-object admin API.
|
||||
type HealResult struct {
|
||||
State HealState `json:"state"`
|
||||
@@ -590,140 +473,3 @@ func (adm *AdminClient) HealFormat(dryrun bool) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// mkUploadsHealQuery - helper function to construct query params for
|
||||
// ListUploadsHeal API.
|
||||
func mkUploadsHealQuery(bucket, prefix, marker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
|
||||
|
||||
queryVal := make(url.Values)
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(healBucket), bucket)
|
||||
queryVal.Set(string(healPrefix), prefix)
|
||||
queryVal.Set(string(healMarker), marker)
|
||||
queryVal.Set(string(healUploadIDMarker), uploadIDMarker)
|
||||
queryVal.Set(string(healDelimiter), delimiter)
|
||||
queryVal.Set(string(healMaxUpload), maxUploadsStr)
|
||||
return queryVal
|
||||
}
|
||||
|
||||
func (adm *AdminClient) listUploadsHeal(bucket, prefix, marker, uploadIDMarker, delimiter string, maxUploads int) (listUploadsHealResponse, error) {
|
||||
// Construct query params.
|
||||
maxUploadsStr := fmt.Sprintf("%d", maxUploads)
|
||||
queryVal := mkUploadsHealQuery(bucket, prefix, marker, uploadIDMarker, delimiter, maxUploadsStr)
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "list-uploads")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Empty 'list' of objects to be healed.
|
||||
toBeHealedUploads := listUploadsHealResponse{}
|
||||
|
||||
// Execute GET on /?heal to list objects needing heal.
|
||||
resp, err := adm.executeMethod("GET", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return listUploadsHealResponse{}, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return toBeHealedUploads, httpRespToErrorResponse(resp)
|
||||
|
||||
}
|
||||
|
||||
err = xml.NewDecoder(resp.Body).Decode(&toBeHealedUploads)
|
||||
if err != nil {
|
||||
return listUploadsHealResponse{}, err
|
||||
}
|
||||
|
||||
return toBeHealedUploads, nil
|
||||
}
|
||||
|
||||
// ListUploadsHeal - issues list heal uploads API request
|
||||
func (adm *AdminClient) ListUploadsHeal(bucket, prefix string, recursive bool,
|
||||
doneCh <-chan struct{}) (<-chan UploadInfo, error) {
|
||||
|
||||
// Default listing is delimited at "/"
|
||||
delimiter := "/"
|
||||
if recursive {
|
||||
// If recursive we do not delimit.
|
||||
delimiter = ""
|
||||
}
|
||||
|
||||
uploadIDMarker := ""
|
||||
|
||||
// Allocate new list objects channel.
|
||||
uploadStatCh := make(chan UploadInfo, maxUploadsList)
|
||||
|
||||
// Initiate list objects goroutine here.
|
||||
go func(uploadStatCh chan<- UploadInfo) {
|
||||
defer close(uploadStatCh)
|
||||
// Save marker for next request.
|
||||
var marker string
|
||||
for {
|
||||
// Get list of objects a maximum of 1000 per request.
|
||||
result, err := adm.listUploadsHeal(bucket, prefix, marker,
|
||||
uploadIDMarker, delimiter, maxUploadsList)
|
||||
if err != nil {
|
||||
uploadStatCh <- UploadInfo{
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// If contents are available loop through and
|
||||
// send over channel.
|
||||
for _, upload := range result.Uploads {
|
||||
select {
|
||||
// Send upload info.
|
||||
case uploadStatCh <- UploadInfo{
|
||||
Key: upload.Key,
|
||||
UploadID: upload.UploadID,
|
||||
Initiated: upload.Initiated,
|
||||
HealUploadInfo: upload.HealUploadInfo,
|
||||
}:
|
||||
// If receives done from the caller, return here.
|
||||
case <-doneCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Send all common prefixes if any. NOTE:
|
||||
// prefixes are only present if the request is
|
||||
// delimited.
|
||||
for _, prefix := range result.CommonPrefixes {
|
||||
upload := UploadInfo{}
|
||||
upload.Key = prefix.Prefix
|
||||
select {
|
||||
// Send object prefixes.
|
||||
case uploadStatCh <- upload:
|
||||
// If receives done from the caller, return here.
|
||||
case <-doneCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// If next uploadID marker is present, set it
|
||||
// for the next request.
|
||||
if result.NextUploadIDMarker != "" {
|
||||
uploadIDMarker = result.NextUploadIDMarker
|
||||
}
|
||||
|
||||
// If next marker present, save it for next request.
|
||||
if result.KeyMarker != "" {
|
||||
marker = result.KeyMarker
|
||||
}
|
||||
|
||||
// Listing ends result is not truncated,
|
||||
// return right here.
|
||||
if !result.IsTruncated {
|
||||
return
|
||||
}
|
||||
}
|
||||
}(uploadStatCh)
|
||||
return uploadStatCh, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user