2016-08-30 22:22:27 -04:00
/ *
2019-04-09 14:39:42 -04:00
* MinIO Cloud Storage , ( C ) 2016 MinIO , Inc .
2016-08-30 22:22:27 -04:00
*
* Licensed under the Apache License , Version 2.0 ( the "License" ) ;
* you may not use this file except in compliance with the License .
* You may obtain a copy of the License at
*
* http : //www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing , software
* distributed under the License is distributed on an "AS IS" BASIS ,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
* See the License for the specific language governing permissions and
* limitations under the License .
* /
package cmd
import (
2018-04-05 18:04:40 -04:00
"context"
2020-01-14 21:45:17 -05:00
"crypto/tls"
2017-06-17 14:20:12 -04:00
"fmt"
2020-01-14 21:45:17 -05:00
"net/http"
"net/url"
2018-02-15 20:45:57 -05:00
"os"
2020-01-14 21:45:17 -05:00
"path"
2019-10-30 03:04:39 -04:00
"sync"
2016-08-30 22:22:27 -04:00
"time"
2020-01-14 21:45:17 -05:00
xhttp "github.com/minio/minio/cmd/http"
2018-04-05 18:04:40 -04:00
"github.com/minio/minio/cmd/logger"
2020-01-14 21:45:17 -05:00
"github.com/minio/minio/cmd/rest"
2019-01-25 16:33:28 -05:00
"github.com/minio/minio/pkg/sync/errgroup"
2016-08-30 22:22:27 -04:00
)
2018-02-15 20:45:57 -05:00
var printEndpointError = func ( ) func ( Endpoint , error ) {
2019-10-30 03:04:39 -04:00
var mutex sync . Mutex
2018-02-15 20:45:57 -05:00
printOnce := make ( map [ Endpoint ] map [ string ] bool )
return func ( endpoint Endpoint , err error ) {
2018-09-14 00:42:50 -04:00
reqInfo := ( & logger . ReqInfo { } ) . AppendTags ( "endpoint" , endpoint . String ( ) )
2018-04-05 18:04:40 -04:00
ctx := logger . SetReqInfo ( context . Background ( ) , reqInfo )
2019-10-30 03:04:39 -04:00
mutex . Lock ( )
defer mutex . Unlock ( )
2018-02-15 20:45:57 -05:00
m , ok := printOnce [ endpoint ]
if ! ok {
m = make ( map [ string ] bool )
m [ err . Error ( ) ] = true
printOnce [ endpoint ] = m
2018-08-14 16:58:48 -04:00
logger . LogAlwaysIf ( ctx , err )
2018-02-15 20:45:57 -05:00
return
2017-12-28 12:32:48 -05:00
}
2018-02-15 20:45:57 -05:00
if m [ err . Error ( ) ] {
return
2017-12-28 12:32:48 -05:00
}
2018-02-15 20:45:57 -05:00
m [ err . Error ( ) ] = true
2018-08-14 16:58:48 -04:00
logger . LogAlwaysIf ( ctx , err )
2017-12-28 12:32:48 -05:00
}
2018-02-15 20:45:57 -05:00
} ( )
2016-08-30 22:22:27 -04:00
2018-03-15 16:55:23 -04:00
// Migrates backend format of local disks.
2019-11-19 20:42:27 -05:00
func formatXLMigrateLocalEndpoints ( endpoints Endpoints ) error {
2019-01-25 16:33:28 -05:00
g := errgroup . WithNErrs ( len ( endpoints ) )
for index , endpoint := range endpoints {
2018-02-15 20:45:57 -05:00
if ! endpoint . IsLocal {
continue
2016-08-30 22:22:27 -04:00
}
2019-01-25 16:33:28 -05:00
index := index
g . Go ( func ( ) error {
epPath := endpoints [ index ] . Path
formatPath := pathJoin ( epPath , minioMetaBucket , formatConfigFile )
if _ , err := os . Stat ( formatPath ) ; err != nil {
if os . IsNotExist ( err ) {
return nil
}
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "unable to access (%s) %w" , formatPath , err )
2018-02-15 20:45:57 -05:00
}
2019-01-25 16:33:28 -05:00
return formatXLMigrate ( epPath )
} , index )
}
for _ , err := range g . Wait ( ) {
if err != nil {
2018-02-15 20:45:57 -05:00
return err
2016-11-23 18:48:10 -05:00
}
}
2018-02-15 20:45:57 -05:00
return nil
2016-11-23 18:48:10 -05:00
}
2018-03-15 16:55:23 -04:00
// Cleans up tmp directory of local disks.
2019-11-19 20:42:27 -05:00
func formatXLCleanupTmpLocalEndpoints ( endpoints Endpoints ) error {
2019-01-25 16:33:28 -05:00
g := errgroup . WithNErrs ( len ( endpoints ) )
for index , endpoint := range endpoints {
2018-03-15 16:55:23 -04:00
if ! endpoint . IsLocal {
continue
}
2019-01-25 16:33:28 -05:00
index := index
g . Go ( func ( ) error {
epPath := endpoints [ index ] . Path
// If disk is not formatted there is nothing to be cleaned up.
formatPath := pathJoin ( epPath , minioMetaBucket , formatConfigFile )
if _ , err := os . Stat ( formatPath ) ; err != nil {
if os . IsNotExist ( err ) {
return nil
}
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "unable to access (%s) %w" , formatPath , err )
2018-03-15 16:55:23 -04:00
}
2019-01-25 16:33:28 -05:00
if _ , err := os . Stat ( pathJoin ( epPath , minioMetaTmpBucket + "-old" ) ) ; err != nil {
if ! os . IsNotExist ( err ) {
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "unable to access (%s) %w" ,
2019-08-05 14:41:29 -04:00
pathJoin ( epPath , minioMetaTmpBucket + "-old" ) ,
err )
2019-01-25 16:33:28 -05:00
}
}
// Need to move temporary objects left behind from previous run of minio
// server to a unique directory under `minioMetaTmpBucket-old` to clean
// up `minioMetaTmpBucket` for the current run.
//
// /disk1/.minio.sys/tmp-old/
// |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
// |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
//
// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
// temporary objects from one of the previous runs of minio server.
2019-08-05 14:41:29 -04:00
tmpOld := pathJoin ( epPath , minioMetaTmpBucket + "-old" , mustGetUUID ( ) )
2019-01-25 16:33:28 -05:00
if err := renameAll ( pathJoin ( epPath , minioMetaTmpBucket ) ,
2019-08-05 14:41:29 -04:00
tmpOld ) ; err != nil && err != errFileNotFound {
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "unable to rename (%s -> %s) %w" ,
2019-08-05 14:41:29 -04:00
pathJoin ( epPath , minioMetaTmpBucket ) ,
tmpOld ,
err )
2019-01-25 16:33:28 -05:00
}
// Removal of tmp-old folder is backgrounded completely.
go removeAll ( pathJoin ( epPath , minioMetaTmpBucket + "-old" ) )
2019-08-05 14:41:29 -04:00
if err := mkdirAll ( pathJoin ( epPath , minioMetaTmpBucket ) , 0777 ) ; err != nil {
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "unable to create (%s) %w" ,
2019-08-05 14:41:29 -04:00
pathJoin ( epPath , minioMetaTmpBucket ) ,
err )
}
return nil
2019-01-25 16:33:28 -05:00
} , index )
}
for _ , err := range g . Wait ( ) {
if err != nil {
2018-03-15 16:55:23 -04:00
return err
}
}
return nil
}
2018-04-04 00:58:48 -04:00
// validate reference format against list of XL formats.
2019-11-19 20:42:27 -05:00
func validateXLFormats ( format * formatXLV3 , formats [ ] * formatXLV3 , endpoints Endpoints , setCount , drivesPerSet int ) error {
2018-04-04 00:58:48 -04:00
for i := range formats {
if formats [ i ] == nil {
continue
}
if err := formatXLV3Check ( format , formats [ i ] ) ; err != nil {
2019-12-02 12:28:01 -05:00
return fmt . Errorf ( "%s format error: %w" , endpoints [ i ] , err )
2018-04-04 00:58:48 -04:00
}
}
if len ( format . XL . Sets ) != setCount {
return fmt . Errorf ( "Current backend format is inconsistent with input args (%s), Expected set count %d, got %d" , endpoints , len ( format . XL . Sets ) , setCount )
}
if len ( format . XL . Sets [ 0 ] ) != drivesPerSet {
return fmt . Errorf ( "Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d" , endpoints , len ( format . XL . Sets [ 0 ] ) , drivesPerSet )
}
2020-01-13 16:09:10 -05:00
2018-04-04 00:58:48 -04:00
return nil
}
// Following error message is added to fix a regression in release
// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This
// migration failed to capture '.This' field properly which indicates
// the disk UUID association. Below error message is returned when
// we see this situation in format.json, for more info refer
// https://github.com/minio/minio/issues/5667
var errXLV3ThisEmpty = fmt . Errorf ( "XL format version 3 has This field empty" )
2020-01-14 21:45:17 -05:00
// IsServerResolvable - checks if the endpoint is resolvable
// by sending a naked HTTP request with liveness checks.
func IsServerResolvable ( endpoint Endpoint ) error {
serverURL := & url . URL {
Scheme : endpoint . Scheme ,
Host : endpoint . Host ,
Path : path . Join ( healthCheckPathPrefix , healthCheckLivenessPath ) ,
}
var tlsConfig * tls . Config
if globalIsSSL {
tlsConfig = & tls . Config {
ServerName : endpoint . Hostname ( ) ,
RootCAs : globalRootCAs ,
NextProtos : [ ] string { "http/1.1" } , // Force http1.1
}
}
req , err := http . NewRequest ( http . MethodGet , serverURL . String ( ) , nil )
if err != nil {
return err
}
httpClient := & http . Client {
Transport : newCustomHTTPTransport ( tlsConfig , rest . DefaultRESTTimeout , rest . DefaultRESTTimeout ) ( ) ,
}
resp , err := httpClient . Do ( req )
if err != nil {
return err
}
defer xhttp . DrainBody ( resp . Body )
if resp . StatusCode != http . StatusOK {
return StorageErr ( resp . Status )
}
return nil
}
2018-04-04 00:58:48 -04:00
// connect to list of endpoints and load all XL disk formats, validate the formats are correct
// and are in quorum, if no formats are found attempt to initialize all of them for the first
// time. additionally make sure to close all the disks used in this attempt.
2020-01-17 18:39:07 -05:00
func connectLoadInitFormats ( retryCount int , firstDisk bool , endpoints Endpoints , zoneCount , setCount , drivesPerSet int , deploymentID string ) ( * formatXLV3 , error ) {
2018-09-10 19:21:59 -04:00
// Initialize all storage disks
2019-09-27 19:47:12 -04:00
storageDisks , errs := initStorageDisksWithErrors ( endpoints )
2018-04-04 00:58:48 -04:00
defer closeStorageDisks ( storageDisks )
2020-01-14 21:45:17 -05:00
2019-09-27 19:47:12 -04:00
for i , err := range errs {
2020-01-14 21:45:17 -05:00
if err != nil {
if err != errDiskNotFound {
return nil , fmt . Errorf ( "Disk %s: %w" , endpoints [ i ] , err )
}
if retryCount >= 5 {
logger . Info ( "Unable to connect to %s: %v\n" , endpoints [ i ] , IsServerResolvable ( endpoints [ i ] ) )
}
2019-09-27 19:47:12 -04:00
}
}
2018-04-04 00:58:48 -04:00
2018-12-04 13:25:56 -05:00
// Attempt to load all `format.json` from all disks.
formatConfigs , sErrs := loadFormatXLAll ( storageDisks )
// Check if we have
for i , sErr := range sErrs {
if _ , ok := formatCriticalErrors [ sErr ] ; ok {
2019-12-02 12:28:01 -05:00
return nil , fmt . Errorf ( "Disk %s: %w" , endpoints [ i ] , sErr )
2018-12-04 13:25:56 -05:00
}
}
2018-04-04 00:58:48 -04:00
// Pre-emptively check if one of the formatted disks
// is invalid. This function returns success for the
// most part unless one of the formats is not consistent
// with expected XL format. For example if a user is
// trying to pool FS backend into an XL set.
2019-09-27 19:47:12 -04:00
if err := checkFormatXLValues ( formatConfigs ) ; err != nil {
2018-04-04 00:58:48 -04:00
return nil , err
}
2018-04-12 18:43:38 -04:00
// All disks report unformatted we should initialized everyone.
if shouldInitXLDisks ( sErrs ) && firstDisk {
2020-01-17 18:39:07 -05:00
logger . Info ( "Formatting %v zone, %v set(s), %v drives per set." ,
zoneCount , setCount , drivesPerSet )
2019-04-02 13:50:13 -04:00
// Initialize erasure code format on disks
2019-11-20 05:09:30 -05:00
format , err := initFormatXL ( context . Background ( ) , storageDisks , setCount , drivesPerSet , deploymentID )
2019-04-02 13:50:13 -04:00
if err != nil {
return nil , err
}
// Assign globalDeploymentID on first run for the
// minio server managing the first disk
globalDeploymentID = format . ID
2019-12-23 19:31:03 -05:00
return format , nil
}
2018-04-12 18:43:38 -04:00
// Return error when quorum unformatted disks - indicating we are
// waiting for first server to be online.
if quorumUnformattedDisks ( sErrs ) && ! firstDisk {
return nil , errNotFirstDisk
}
// Return error when quorum unformatted disks but waiting for rest
// of the servers to be online.
if quorumUnformattedDisks ( sErrs ) && firstDisk {
return nil , errFirstDiskWait
}
2018-04-04 00:58:48 -04:00
// Following function is added to fix a regressions which was introduced
// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
// This migration failed to capture '.This' field properly which indicates
// the disk UUID association. Below function is called to handle and fix
// this regression, for more info refer https://github.com/minio/minio/issues/5667
2019-09-27 19:47:12 -04:00
if err := fixFormatXLV3 ( storageDisks , endpoints , formatConfigs ) ; err != nil {
2018-04-04 00:58:48 -04:00
return nil , err
}
// If any of the .This field is still empty, we return error.
if formatXLV3ThisEmpty ( formatConfigs ) {
return nil , errXLV3ThisEmpty
}
format , err := getFormatXLInQuorum ( formatConfigs )
if err != nil {
return nil , err
}
// Validate all format configs with reference format.
if err = validateXLFormats ( format , formatConfigs , endpoints , setCount , drivesPerSet ) ; err != nil {
return nil , err
}
2018-07-18 23:17:35 -04:00
// Get the deploymentID if set.
format . ID , err = formatXLGetDeploymentID ( format , formatConfigs )
if err != nil {
return nil , err
}
if format . ID == "" {
2019-11-13 15:17:45 -05:00
// Not a first disk, wait until first disk fixes deploymentID
if ! firstDisk {
return nil , errNotFirstDisk
}
if err = formatXLFixDeploymentID ( endpoints , storageDisks , format ) ; err != nil {
2018-07-18 23:17:35 -04:00
return nil , err
}
}
2018-11-19 17:47:03 -05:00
globalDeploymentID = format . ID
2018-07-18 23:17:35 -04:00
2019-11-13 15:17:45 -05:00
if err = formatXLFixLocalDeploymentID ( endpoints , storageDisks , format ) ; err != nil {
2018-07-18 23:17:35 -04:00
return nil , err
}
2019-12-23 19:31:03 -05:00
2020-01-15 15:36:52 -05:00
// The will always recreate some directories inside .minio.sys of
// the local disk such as tmp, multipart and background-ops
initXLMetaVolumesInLocalDisks ( storageDisks , formatConfigs )
2018-04-04 00:58:48 -04:00
return format , nil
}
2018-02-15 20:45:57 -05:00
// Format disks before initialization of object layer.
2020-01-17 18:39:07 -05:00
func waitForFormatXL ( firstDisk bool , endpoints Endpoints , zoneCount , setCount , drivesPerSet int , deploymentID string ) ( format * formatXLV3 , err error ) {
2019-11-21 07:24:51 -05:00
if len ( endpoints ) == 0 || setCount == 0 || drivesPerSet == 0 {
2018-02-15 20:45:57 -05:00
return nil , errInvalidArgument
2016-11-02 11:51:06 -04:00
}
2018-02-15 20:45:57 -05:00
if err = formatXLMigrateLocalEndpoints ( endpoints ) ; err != nil {
return nil , err
2016-08-30 22:22:27 -04:00
}
2018-03-15 16:55:23 -04:00
if err = formatXLCleanupTmpLocalEndpoints ( endpoints ) ; err != nil {
return nil , err
}
2016-12-07 13:22:00 -05:00
// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
// All times are rounded to avoid showing milli, micro and nano seconds
formatStartTime := time . Now ( ) . Round ( time . Second )
getElapsedTime := func ( ) string {
return time . Now ( ) . Round ( time . Second ) . Sub ( formatStartTime ) . String ( )
}
2020-01-08 16:36:54 -05:00
// Wait on each try for an update.
ticker := time . NewTicker ( 500 * time . Millisecond )
defer ticker . Stop ( )
var tries int
2016-11-02 18:27:36 -04:00
for {
select {
2020-01-08 16:36:54 -05:00
case <- ticker . C :
2020-01-17 18:39:07 -05:00
format , err := connectLoadInitFormats ( tries , firstDisk , endpoints , zoneCount , setCount , drivesPerSet , deploymentID )
2018-04-04 00:58:48 -04:00
if err != nil {
2020-01-08 16:36:54 -05:00
tries ++
2018-04-04 00:58:48 -04:00
switch err {
case errNotFirstDisk :
// Fresh setup, wait for first server to be up.
2018-04-10 12:37:14 -04:00
logger . Info ( "Waiting for the first server to format the disks." )
2017-04-18 13:35:17 -04:00
continue
2018-04-12 18:43:38 -04:00
case errFirstDiskWait :
// Fresh setup, wait for other servers to come up.
logger . Info ( "Waiting for all other servers to be online to format the disks." )
continue
2018-04-04 00:58:48 -04:00
case errXLReadQuorum :
// no quorum available continue to wait for minimum number of servers.
2018-04-10 12:37:14 -04:00
logger . Info ( "Waiting for a minimum of %d disks to come online (elapsed %s)\n" , len ( endpoints ) / 2 , getElapsedTime ( ) )
2018-04-04 00:58:48 -04:00
continue
case errXLV3ThisEmpty :
// need to wait for this error to be healed, so continue.
continue
default :
// For all other unhandled errors we exit and fail.
return nil , err
2017-04-18 13:35:17 -04:00
}
2016-12-11 18:18:55 -05:00
}
2018-04-04 00:58:48 -04:00
return format , nil
2018-02-06 18:07:17 -05:00
case <- globalOSSignalCh :
2018-02-15 20:45:57 -05:00
return nil , fmt . Errorf ( "Initializing data volumes gracefully stopped" )
2016-11-02 18:27:36 -04:00
}
}
2016-08-30 22:22:27 -04:00
}