minio/cmd/prepare-storage.go

/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"os"
	"time"

	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/errors"
)

var printEndpointError = func() func(Endpoint, error) {
	printOnce := make(map[Endpoint]map[string]bool)

	return func(endpoint Endpoint, err error) {
		m, ok := printOnce[endpoint]
		if !ok {
			m = make(map[string]bool)
			m[err.Error()] = true
			printOnce[endpoint] = m
			errorIf(err, "%s: %s", endpoint, err)
			return
		}
		if m[err.Error()] {
			return
		}
		m[err.Error()] = true
		errorIf(err, "%s: %s", endpoint, err)
	}
}()

// Migrates backend format of local disks.
func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
	for _, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
		if _, err := os.Stat(formatPath); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return err
		}
		if err := formatXLMigrate(endpoint.Path); err != nil {
			return err
		}
	}
	return nil
}

// Cleans up tmp directory of local disks.
func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
	for _, endpoint := range endpoints {
		if !endpoint.IsLocal {
			continue
		}
		formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
		if _, err := os.Stat(formatPath); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return err
		}
		if err := os.RemoveAll(pathJoin(endpoint.Path, minioMetaTmpBucket)); err != nil {
			return err
		}
		if err := os.MkdirAll(pathJoin(endpoint.Path, minioMetaTmpBucket), 0777); err != nil {
			return err
		}
	}
	return nil
}

// Format disks before initialization of object layer.
func waitForFormatXL(firstDisk bool, endpoints EndpointList, setCount, disksPerSet int) (format *formatXLV3, err error) {
	if len(endpoints) == 0 || setCount == 0 || disksPerSet == 0 {
		return nil, errInvalidArgument
	}

	if err = formatXLMigrateLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	if err = formatXLCleanupTmpLocalEndpoints(endpoints); err != nil {
		return nil, err
	}

	// Done channel is used to close any lingering retry routine, as soon
	// as this function returns.
	doneCh := make(chan struct{})

	// Indicate to our retry routine to exit cleanly, upon this function return.
	defer close(doneCh)

	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
	// All times are rounded to avoid showing milli, micro and nano seconds
	formatStartTime := time.Now().Round(time.Second)
	getElapsedTime := func() string {
		return time.Now().Round(time.Second).Sub(formatStartTime).String()
	}

	// Wait on the jitter retry loop.
	retryTimerCh := newRetryTimerSimple(doneCh)
	for {
		select {
		case _ = <-retryTimerCh:
			// Attempt to load all `format.json` from all disks.
			formatConfigs, sErrs := loadFormatXLAll(endpoints)

			// Pre-emptively check if one of the formatted disks
			// is invalid. This function returns success for the
			// most part unless one of the formats is not consistent
			// with expected XL format. For example if a user is
			// trying to pool FS backend into an XL set.
			if err = checkFormatXLValues(formatConfigs); err != nil {
				return nil, err
			}

			for i, sErr := range sErrs {
				if _, ok := formatCriticalErrors[errors.Cause(sErr)]; ok {
					return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
				}
			}

			if shouldInitXLDisks(sErrs) {
				if !firstDisk {
					console.Println("Waiting for the first server to format the disks.")
					continue
				}
				return initFormatXL(endpoints, setCount, disksPerSet)
			}

			// Following function is added to fix a regressions which was introduced
			// in release RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3.
			// This migration failed to capture '.This' field properly which indicates
			// the disk UUID association. Below function is called to handle and fix
			// this regression, for more info refer https://github.com/minio/minio/issues/5667
			if err = fixFormatXLV3(endpoints, formatConfigs); err != nil {
				return nil, err
			}

			// If any of the .This field is still empty we wait them to be fixed.
			if formatXLV3ThisEmpty(formatConfigs) {
				continue
			}

			format, err = getFormatXLInQuorum(formatConfigs)
			if err == nil {
				for i := range formatConfigs {
					if formatConfigs[i] == nil {
						continue
					}
					if err = formatXLV3Check(format, formatConfigs[i]); err != nil {
						return nil, fmt.Errorf("%s format error: %s", endpoints[i], err)
					}
				}
				if len(format.XL.Sets) != globalXLSetCount {
					return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected set count %d, got %d", endpoints, len(format.XL.Sets), globalXLSetCount)
				}
				if len(format.XL.Sets[0]) != globalXLSetDriveCount {
					return nil, fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), globalXLSetDriveCount)
				}
				return format, nil
			}
			console.Printf("Waiting for a minimum of %d disks to come online (elapsed %s)\n", len(endpoints)/2, getElapsedTime())
		case <-globalOSSignalCh:
			return nil, fmt.Errorf("Initializing data volumes gracefully stopped")
		}
	}
}