mirror of
https://github.com/minio/minio.git
synced 2025-11-11 06:20:14 -05:00
Move admin APIs to new path and add redesigned heal APIs (#5351)
- Changes related to moving admin APIs
- admin APIs now have an endpoint under /minio/admin
- admin APIs are now versioned - a new API to server the version is
added at "GET /minio/admin/version" and all API operations have the
path prefix /minio/admin/v1/<operation>
- new service stop API added
- credentials change API is moved to /minio/admin/v1/config/credential
- credentials change API and configuration get/set API now require TLS
so that credentials are protected
- all API requests now receive JSON
- heal APIs are disabled as they will be changed substantially
- Heal API changes
Heal API is now provided at a single endpoint with the ability for a
client to start a heal sequence on all the data in the server, a
single bucket, or under a prefix within a bucket.
When a heal sequence is started, the server returns a unique token
that needs to be used for subsequent 'status' requests to fetch heal
results.
On each status request from the client, the server returns heal result
records that it has accumulated since the previous status request. The
server accumulates upto 1000 records and pauses healing further
objects until the client requests for status. If the client does not
request any further records for a long time, the server aborts the
heal sequence automatically.
A heal result record is returned for each entity healed on the server,
such as system metadata, object metadata, buckets and objects, and has
information about the before and after states on each disk.
A client may request to force restart a heal sequence - this causes
the running heal sequence to be aborted at the next safe spot and
starts a new heal sequence.
This commit is contained in:
Aditya Manthramurthy
committed by
Harshavardhana
Harshavardhana
parent
f3f09ed14e
commit
a337ea4d11
@@ -20,456 +20,157 @@
|
||||
package madmin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
// listBucketHealResult container for listObjects response.
|
||||
type listBucketHealResult struct {
|
||||
// A response can contain CommonPrefixes only if you have
|
||||
// specified a delimiter.
|
||||
CommonPrefixes []commonPrefix
|
||||
// Metadata about each object returned.
|
||||
Contents []ObjectInfo
|
||||
Delimiter string
|
||||
|
||||
// Encoding type used to encode object keys in the response.
|
||||
EncodingType string
|
||||
|
||||
// A flag that indicates whether or not ListObjects returned all of the results
|
||||
// that satisfied the search criteria.
|
||||
IsTruncated bool
|
||||
Marker string
|
||||
MaxKeys int64
|
||||
Name string
|
||||
|
||||
// When response is truncated (the IsTruncated element value in
|
||||
// the response is true), you can use the key name in this field
|
||||
// as marker in the subsequent request to get next set of objects.
|
||||
// Object storage lists objects in alphabetical order Note: This
|
||||
// element is returned only if you have delimiter request
|
||||
// parameter specified. If response does not include the NextMaker
|
||||
// and it is truncated, you can use the value of the last Key in
|
||||
// the response as the marker in the subsequent request to get the
|
||||
// next set of object keys.
|
||||
NextMarker string
|
||||
Prefix string
|
||||
// HealOpts - collection of options for a heal sequence
|
||||
type HealOpts struct {
|
||||
Recursive bool `json:"recursive"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
}
|
||||
|
||||
// commonPrefix container for prefix response.
|
||||
type commonPrefix struct {
|
||||
Prefix string
|
||||
// HealStartSuccess - holds information about a successfully started
|
||||
// heal operation
|
||||
type HealStartSuccess struct {
|
||||
ClientToken string `json:"clientToken"`
|
||||
ClientAddress string `json:"clientAddress"`
|
||||
StartTime time.Time `json:"startTime"`
|
||||
}
|
||||
|
||||
// Owner - bucket owner/principal
|
||||
type Owner struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
// HealTaskStatus - status struct for a heal task
|
||||
type HealTaskStatus struct {
|
||||
Summary string `json:"summary"`
|
||||
FailureDetail string `json:"detail"`
|
||||
StartTime time.Time `json:"startTime"`
|
||||
HealSettings HealOpts `json:"settings"`
|
||||
NumDisks int `json:"numDisks"`
|
||||
|
||||
Items []HealResultItem `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
// Bucket container for bucket metadata
|
||||
type Bucket struct {
|
||||
Name string
|
||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
|
||||
HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"`
|
||||
}
|
||||
|
||||
// ListBucketsHealResponse - format for list buckets response
|
||||
type ListBucketsHealResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
|
||||
|
||||
Owner Owner
|
||||
|
||||
// Container for one or more buckets.
|
||||
Buckets struct {
|
||||
Buckets []Bucket `xml:"Bucket"`
|
||||
} // Buckets are nested
|
||||
}
|
||||
|
||||
// HealStatus - represents different states of healing an object could be in.
|
||||
type HealStatus int
|
||||
// HealItemType - specify the type of heal operation in a healing
|
||||
// result
|
||||
type HealItemType string
|
||||
|
||||
// HealItemType constants
|
||||
const (
|
||||
// Healthy - Object that is already healthy
|
||||
Healthy HealStatus = iota
|
||||
// CanHeal - Object can be healed
|
||||
CanHeal
|
||||
// Corrupted - Object can't be healed
|
||||
Corrupted
|
||||
// QuorumUnavailable - Object can't be healed until read
|
||||
// quorum is available
|
||||
QuorumUnavailable
|
||||
// CanPartiallyHeal - Object can't be healed completely until
|
||||
// disks with missing parts come online
|
||||
CanPartiallyHeal
|
||||
HealItemMetadata HealItemType = "metadata"
|
||||
HealItemBucket = "bucket"
|
||||
HealItemBucketMetadata = "bucket-metadata"
|
||||
HealItemObject = "object"
|
||||
)
|
||||
|
||||
// HealBucketInfo - represents healing related information of a bucket.
|
||||
type HealBucketInfo struct {
|
||||
Status HealStatus
|
||||
}
|
||||
|
||||
// BucketInfo - represents bucket metadata.
|
||||
type BucketInfo struct {
|
||||
// Name of the bucket.
|
||||
Name string
|
||||
|
||||
// Date and time when the bucket was created.
|
||||
Created time.Time
|
||||
|
||||
// Healing information
|
||||
HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"`
|
||||
}
|
||||
|
||||
// HealObjectInfo - represents healing related information of an object.
|
||||
type HealObjectInfo struct {
|
||||
Status HealStatus
|
||||
MissingDataCount int
|
||||
MissingParityCount int
|
||||
}
|
||||
|
||||
// ObjectInfo container for object metadata.
|
||||
type ObjectInfo struct {
|
||||
// An ETag is optionally set to md5sum of an object. In case of multipart objects,
|
||||
// ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of
|
||||
// each parts concatenated into one string.
|
||||
ETag string `json:"etag"`
|
||||
|
||||
Key string `json:"name"` // Name of the object
|
||||
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
|
||||
Size int64 `json:"size"` // Size in bytes of the object.
|
||||
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
|
||||
|
||||
// Collection of additional metadata on the object.
|
||||
// eg: x-amz-meta-*, content-encoding etc.
|
||||
Metadata http.Header `json:"metadata"`
|
||||
|
||||
// Owner name.
|
||||
Owner struct {
|
||||
DisplayName string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
} `json:"owner"`
|
||||
|
||||
// The class of storage used to store the object.
|
||||
StorageClass string `json:"storageClass"`
|
||||
|
||||
// Error
|
||||
Err error `json:"-"`
|
||||
HealObjectInfo *HealObjectInfo `json:"healObjectInfo,omitempty"`
|
||||
}
|
||||
|
||||
type healQueryKey string
|
||||
|
||||
// Drive state constants
|
||||
const (
|
||||
healBucket healQueryKey = "bucket"
|
||||
healObject healQueryKey = "object"
|
||||
healPrefix healQueryKey = "prefix"
|
||||
healMarker healQueryKey = "marker"
|
||||
healDelimiter healQueryKey = "delimiter"
|
||||
healMaxKey healQueryKey = "max-key"
|
||||
healDryRun healQueryKey = "dry-run"
|
||||
DriveStateOk string = "ok"
|
||||
DriveStateOffline = "offline"
|
||||
DriveStateCorrupt = "corrupt"
|
||||
DriveStateMissing = "missing"
|
||||
)
|
||||
|
||||
// mkHealQueryVal - helper function to construct heal REST API query params.
|
||||
func mkHealQueryVal(bucket, prefix, marker, delimiter, maxKeyStr string) url.Values {
|
||||
queryVal := make(url.Values)
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(healBucket), bucket)
|
||||
queryVal.Set(string(healPrefix), prefix)
|
||||
queryVal.Set(string(healMarker), marker)
|
||||
queryVal.Set(string(healDelimiter), delimiter)
|
||||
queryVal.Set(string(healMaxKey), maxKeyStr)
|
||||
return queryVal
|
||||
// HealResultItem - struct for an individual heal result item
|
||||
type HealResultItem struct {
|
||||
ResultIndex int64 `json:"resultId"`
|
||||
Type HealItemType `json:"type"`
|
||||
Bucket string `json:"bucket"`
|
||||
Object string `json:"object"`
|
||||
Detail string `json:"detail"`
|
||||
ParityBlocks int `json:"parityBlocks,omitempty"`
|
||||
DataBlocks int `json:"dataBlocks,omitempty"`
|
||||
DiskCount int `json:"diskCount"`
|
||||
DriveInfo struct {
|
||||
// below maps are from drive endpoint to drive state
|
||||
Before map[string]string `json:"before"`
|
||||
After map[string]string `json:"after"`
|
||||
} `json:"drives"`
|
||||
ObjectSize int64 `json:"objectSize"`
|
||||
}
|
||||
|
||||
// listObjectsHeal - issues heal list API request for a batch of maxKeys objects to be healed.
|
||||
func (adm *AdminClient) listObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (listBucketHealResult, error) {
|
||||
// Construct query params.
|
||||
maxKeyStr := fmt.Sprintf("%d", maxKeys)
|
||||
queryVal := mkHealQueryVal(bucket, prefix, marker, delimiter, maxKeyStr)
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "list-objects")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Empty 'list' of objects to be healed.
|
||||
toBeHealedObjects := listBucketHealResult{}
|
||||
|
||||
// Execute GET on /?heal to list objects needing heal.
|
||||
resp, err := adm.executeMethod("GET", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return listBucketHealResult{}, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return toBeHealedObjects, httpRespToErrorResponse(resp)
|
||||
|
||||
}
|
||||
|
||||
err = xml.NewDecoder(resp.Body).Decode(&toBeHealedObjects)
|
||||
return toBeHealedObjects, err
|
||||
// InitDrives - initialize maps used to represent drive info
|
||||
func (hri *HealResultItem) InitDrives() {
|
||||
hri.DriveInfo.Before = make(map[string]string)
|
||||
hri.DriveInfo.After = make(map[string]string)
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Lists upto maxKeys objects that needing heal matching bucket, prefix, marker, delimiter.
|
||||
func (adm *AdminClient) ListObjectsHeal(bucket, prefix string, recursive bool, doneCh <-chan struct{}) (<-chan ObjectInfo, error) {
|
||||
// Allocate new list objects channel.
|
||||
objectStatCh := make(chan ObjectInfo, 1)
|
||||
// Default listing is delimited at "/"
|
||||
delimiter := "/"
|
||||
if recursive {
|
||||
// If recursive we do not delimit.
|
||||
delimiter = ""
|
||||
// GetOnlineCounts - returns the number of online disks before and
|
||||
// after heal
|
||||
func (hri *HealResultItem) GetOnlineCounts() (b, a int) {
|
||||
if hri == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Initiate list objects goroutine here.
|
||||
go func(objectStatCh chan<- ObjectInfo) {
|
||||
defer close(objectStatCh)
|
||||
// Save marker for next request.
|
||||
var marker string
|
||||
for {
|
||||
// Get list of objects a maximum of 1000 per request.
|
||||
result, err := adm.listObjectsHeal(bucket, prefix, marker, delimiter, 1000)
|
||||
if err != nil {
|
||||
objectStatCh <- ObjectInfo{
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// If contents are available loop through and send over channel.
|
||||
for _, object := range result.Contents {
|
||||
// Save the marker.
|
||||
marker = object.Key
|
||||
select {
|
||||
// Send object content.
|
||||
case objectStatCh <- object:
|
||||
// If receives done from the caller, return here.
|
||||
case <-doneCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Send all common prefixes if any.
|
||||
// NOTE: prefixes are only present if the request is delimited.
|
||||
for _, obj := range result.CommonPrefixes {
|
||||
object := ObjectInfo{}
|
||||
object.Key = obj.Prefix
|
||||
object.Size = 0
|
||||
select {
|
||||
// Send object prefixes.
|
||||
case objectStatCh <- object:
|
||||
// If receives done from the caller, return here.
|
||||
case <-doneCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// If next marker present, save it for next request.
|
||||
if result.NextMarker != "" {
|
||||
marker = result.NextMarker
|
||||
}
|
||||
|
||||
// Listing ends result is not truncated, return right here.
|
||||
if !result.IsTruncated {
|
||||
return
|
||||
}
|
||||
for _, v := range hri.DriveInfo.Before {
|
||||
if v == DriveStateOk {
|
||||
b++
|
||||
}
|
||||
}(objectStatCh)
|
||||
return objectStatCh, nil
|
||||
}
|
||||
|
||||
const timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
|
||||
|
||||
// ListBucketsHeal - issues heal bucket list API request
|
||||
func (adm *AdminClient) ListBucketsHeal() ([]BucketInfo, error) {
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "list-buckets")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Execute GET on /?heal to list objects needing heal.
|
||||
resp, err := adm.executeMethod("GET", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return []BucketInfo{}, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return []BucketInfo{}, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
var listBucketsHealResult ListBucketsHealResponse
|
||||
|
||||
err = xml.NewDecoder(resp.Body).Decode(&listBucketsHealResult)
|
||||
if err != nil {
|
||||
return []BucketInfo{}, err
|
||||
}
|
||||
|
||||
var bucketsToBeHealed []BucketInfo
|
||||
|
||||
for _, bucket := range listBucketsHealResult.Buckets.Buckets {
|
||||
creationDate, err := time.Parse(timeFormatAMZLong, bucket.CreationDate)
|
||||
if err != nil {
|
||||
return []BucketInfo{}, err
|
||||
for _, v := range hri.DriveInfo.After {
|
||||
if v == DriveStateOk {
|
||||
a++
|
||||
}
|
||||
bucketsToBeHealed = append(bucketsToBeHealed,
|
||||
BucketInfo{
|
||||
Name: bucket.Name,
|
||||
Created: creationDate,
|
||||
HealBucketInfo: bucket.HealBucketInfo,
|
||||
})
|
||||
}
|
||||
|
||||
return bucketsToBeHealed, nil
|
||||
return
|
||||
}
|
||||
|
||||
// HealBucket - Heal the given bucket
|
||||
func (adm *AdminClient) HealBucket(bucket string, dryrun bool) error {
|
||||
// Construct query params.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(healBucket), bucket)
|
||||
if dryrun {
|
||||
queryVal.Set(string(healDryRun), "")
|
||||
// Heal - API endpoint to start heal and to fetch status
|
||||
func (adm *AdminClient) Heal(bucket, prefix string, healOpts HealOpts,
|
||||
clientToken string, forceStart bool) (
|
||||
healStart HealStartSuccess, healTaskStatus HealTaskStatus, err error) {
|
||||
|
||||
body, err := json.Marshal(healOpts)
|
||||
if err != nil {
|
||||
return healStart, healTaskStatus, err
|
||||
}
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "bucket")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
path := fmt.Sprintf("/v1/heal/%s", bucket)
|
||||
if bucket != "" && prefix != "" {
|
||||
path += "/" + prefix
|
||||
}
|
||||
|
||||
// Execute POST on /?heal&bucket=mybucket to heal a bucket.
|
||||
resp, err := adm.executeMethod("POST", reqData)
|
||||
// execute POST request to heal api
|
||||
queryVals := make(url.Values)
|
||||
var contentBody io.Reader
|
||||
if clientToken != "" {
|
||||
queryVals.Set("clientToken", clientToken)
|
||||
} else {
|
||||
// Set a body only if clientToken is not given
|
||||
contentBody = bytes.NewReader(body)
|
||||
}
|
||||
if forceStart {
|
||||
queryVals.Set("forceStart", "true")
|
||||
}
|
||||
|
||||
resp, err := adm.executeMethod("POST", requestData{
|
||||
relPath: path,
|
||||
contentBody: contentBody,
|
||||
contentSHA256Bytes: sum256(body),
|
||||
queryValues: queryVals,
|
||||
})
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
return healStart, healTaskStatus, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return httpRespToErrorResponse(resp)
|
||||
return healStart, healTaskStatus, httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HealResult - represents result of heal-object admin API.
|
||||
type HealResult struct {
|
||||
State HealState `json:"state"`
|
||||
}
|
||||
|
||||
// HealState - different states of heal operation
|
||||
type HealState int
|
||||
|
||||
const (
|
||||
// HealNone - none of the disks healed
|
||||
HealNone HealState = iota
|
||||
// HealPartial - some disks were healed, others were offline
|
||||
HealPartial
|
||||
// HealOK - all disks were healed
|
||||
HealOK
|
||||
)
|
||||
|
||||
// HealObject - Heal the given object.
|
||||
func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) (HealResult, error) {
|
||||
// Construct query params.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(healBucket), bucket)
|
||||
queryVal.Set(string(healObject), object)
|
||||
if dryrun {
|
||||
queryVal.Set(string(healDryRun), "")
|
||||
}
|
||||
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "object")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Execute POST on /?heal&bucket=mybucket&object=myobject to heal an object.
|
||||
resp, err := adm.executeMethod("POST", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
respBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
return healStart, healTaskStatus, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return HealResult{}, httpRespToErrorResponse(resp)
|
||||
// Was it a status request?
|
||||
if clientToken == "" {
|
||||
err = json.Unmarshal(respBytes, &healStart)
|
||||
} else {
|
||||
err = json.Unmarshal(respBytes, &healTaskStatus)
|
||||
}
|
||||
|
||||
// Healing is not performed so heal object result is empty.
|
||||
if dryrun {
|
||||
return HealResult{}, nil
|
||||
}
|
||||
|
||||
jsonBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
}
|
||||
|
||||
healResult := HealResult{}
|
||||
err = json.Unmarshal(jsonBytes, &healResult)
|
||||
if err != nil {
|
||||
return HealResult{}, err
|
||||
}
|
||||
|
||||
return healResult, nil
|
||||
}
|
||||
|
||||
// HealFormat - heal storage format on available disks.
|
||||
func (adm *AdminClient) HealFormat(dryrun bool) error {
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
if dryrun {
|
||||
queryVal.Set(string(healDryRun), "")
|
||||
}
|
||||
|
||||
// Set x-minio-operation to format.
|
||||
hdrs := make(http.Header)
|
||||
hdrs.Set(minioAdminOpHeader, "format")
|
||||
|
||||
reqData := requestData{
|
||||
queryValues: queryVal,
|
||||
customHeaders: hdrs,
|
||||
}
|
||||
|
||||
// Execute POST on /?heal to heal storage format.
|
||||
resp, err := adm.executeMethod("POST", reqData)
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return httpRespToErrorResponse(resp)
|
||||
}
|
||||
|
||||
return nil
|
||||
return healStart, healTaskStatus, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user