// Copyright (c) 2015-2022 MinIO, Inc. // // This file is part of MinIO Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "context" "encoding/gob" "errors" "fmt" "io" "net/url" "strconv" xhttp "github.com/minio/minio/internal/http" "github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/rest" "github.com/minio/minio/internal/sync/errgroup" xnet "github.com/minio/pkg/net" ) var errPeerOffline = errors.New("peer is offline") // client to talk to peer Nodes. type peerS3Client struct { host *xnet.Host restClient *rest.Client } // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints() // after verifying format.json func (client *peerS3Client) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) { return client.callWithContext(GlobalContext, method, values, body, length) } // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints() // after verifying format.json func (client *peerS3Client) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) { if values == nil { values = make(url.Values) } respBody, err = client.restClient.Call(ctx, method, values, body, length) if err == nil { return respBody, nil } err = toStorageErr(err) return nil, err } // S3PeerSys - S3 peer call system. type S3PeerSys struct { peerClients []*peerS3Client // Excludes self allPeerClients []*peerS3Client // Includes nil client for self } // NewS3PeerSys - creates new S3 peer calls. func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys { remote, all := newPeerS3Clients(endpoints) return &S3PeerSys{ peerClients: remote, allPeerClients: all, } } // GetBucketInfo returns bucket stat info about bucket on disk across all peers func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (binfo BucketInfo, err error) { g := errgroup.WithNErrs(len(sys.peerClients)) bucketInfos := make([]BucketInfo, len(sys.peerClients)+1) bucketInfo, err := getBucketInfoLocal(ctx, bucket, opts) if err != nil { return BucketInfo{}, err } errs := []error{nil} bucketInfos[0] = bucketInfo for idx, client := range sys.peerClients { idx := idx client := client g.Go(func() error { if client == nil { return errPeerOffline } bucketInfo, err := client.GetBucketInfo(ctx, bucket, opts) if err != nil { return err } bucketInfos[idx] = bucketInfo return nil }, idx) } errs = append(errs, g.Wait()...) quorum := (len(sys.allPeerClients) / 2) if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil { return BucketInfo{}, err } for i, err := range errs { if err == nil { bucketInfo = bucketInfos[i] break } } return bucketInfo, nil } // GetBucketInfo returns bucket stat info from a peer func (client *peerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { v := url.Values{} v.Set(peerS3Bucket, bucket) v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Deleted)) respBody, err := client.call(peerS3MethodGetBucketInfo, v, nil, -1) if err != nil { return BucketInfo{}, err } defer xhttp.DrainBody(respBody) var bucketInfo BucketInfo err = gob.NewDecoder(respBody).Decode(&bucketInfo) return bucketInfo, err } // MakeBucket creates bucket across all peers func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { g := errgroup.WithNErrs(len(sys.peerClients)) for idx, client := range sys.peerClients { client := client g.Go(func() error { if client == nil { return errPeerOffline } return client.MakeBucket(ctx, bucket, opts) }, idx) } errs := g.Wait() errs = append(errs, makeBucketLocal(ctx, bucket, opts)) quorum := (len(sys.allPeerClients) / 2) + 1 err := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum) return toObjectErr(err, bucket) } // MakeBucket creates a bucket on a peer func (client *peerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { v := url.Values{} v.Set(peerS3Bucket, bucket) v.Set(peerS3BucketForceCreate, strconv.FormatBool(opts.ForceCreate)) respBody, err := client.call(peerS3MethodMakeBucket, v, nil, -1) if err != nil { return err } defer xhttp.DrainBody(respBody) return nil } // DeleteBucket deletes bucket across all peers func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { g := errgroup.WithNErrs(len(sys.peerClients)) for idx, client := range sys.peerClients { client := client g.Go(func() error { if client == nil { return errPeerOffline } return client.DeleteBucket(ctx, bucket, opts) }, idx) } errs := g.Wait() errs = append(errs, deleteBucketLocal(ctx, bucket, opts)) for _, err := range errs { if err != nil { return err } } return nil } // DeleteBucket deletes bucket on a peer func (client *peerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { v := url.Values{} v.Set(peerS3Bucket, bucket) v.Set(peerS3BucketForceDelete, strconv.FormatBool(opts.Force)) respBody, err := client.call(peerS3MethodDeleteBucket, v, nil, -1) if err != nil { return err } defer xhttp.DrainBody(respBody) return nil } // newPeerS3Clients creates new peer clients. // The two slices will point to the same clients, // but 'all' will contain nil entry for local client. // The 'all' slice will be in the same order across the cluster. func newPeerS3Clients(endpoints EndpointServerPools) (remote, all []*peerS3Client) { if !globalIsDistErasure { // Only useful in distributed setups return nil, nil } hosts := endpoints.hostsSorted() remote = make([]*peerS3Client, 0, len(hosts)) all = make([]*peerS3Client, len(hosts)) for i, host := range hosts { if host == nil { continue } all[i] = newPeerS3Client(host) remote = append(remote, all[i]) } if len(all) != len(remote)+1 { logger.LogIf(context.Background(), fmt.Errorf("WARNING: Expected number of all hosts (%v) to be remote +1 (%v)", len(all), len(remote))) } return remote, all } // Returns a peer S3 client. func newPeerS3Client(peer *xnet.Host) *peerS3Client { scheme := "http" if globalIsTLS { scheme = "https" } serverURL := &url.URL{ Scheme: scheme, Host: peer.String(), Path: peerS3Path, } restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) // Use a separate client to avoid recursive calls. healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) healthClient.ExpectTimeouts = true healthClient.NoMetrics = true // Construct a new health function. restClient.HealthCheckFn = func() bool { ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout) defer cancel() respBody, err := healthClient.Call(ctx, peerS3MethodHealth, nil, nil, -1) xhttp.DrainBody(respBody) return !isNetworkError(err) } return &peerS3Client{host: peer, restClient: restClient} }