Add a generic registerShutdown function for graceful exit (#2344)

* Add a generic registerShutdown function for graceful exit
* Add shutdown callback test case

fs-v1.go

@@ -63,33 +63,33 @@ func loadFormatFS(storageDisk StorageAPI) (format formatConfigV1, err error) {
 }
 
 // Should be called when process shuts down.
-func shutdownFS(storage StorageAPI) {
+func shutdownFS(storage StorageAPI) errCode {
 	// List if there are any multipart entries.
 	_, err := storage.ListDir(minioMetaBucket, mpartMetaPrefix)
 	if err != errFileNotFound {
 		// Multipart directory is not empty hence do not remove '.minio.sys' volume.
-		os.Exit(0)
+		return exitSuccess
 	}
 	// List if there are any bucket configuration entries.
 	_, err = storage.ListDir(minioMetaBucket, bucketConfigPrefix)
 	if err != errFileNotFound {
 		// Bucket config directory is not empty hence do not remove '.minio.sys' volume.
-		os.Exit(0)
+		return exitSuccess
 	}
 	// Cleanup everything else.
 	prefix := ""
 	if err = cleanupDir(storage, minioMetaBucket, prefix); err != nil {
 		errorIf(err, "Unable to cleanup minio meta bucket")
-		os.Exit(1)
+		return exitFailure
 	}
 	if err = storage.DeleteVol(minioMetaBucket); err != nil {
 		if err != errVolumeNotEmpty {
 			errorIf(err, "Unable to delete minio meta bucket %s", minioMetaBucket)
-			os.Exit(1)
+			return exitFailure
 		}
 	}
 	// Successful exit.
-	os.Exit(0)
+	return exitSuccess
 }
 
 // newFSObjects - initialize new fs object layer.
@@ -133,8 +133,8 @@ func newFSObjects(disk string) (ObjectLayer, error) {
 	}
 
 	// Register the callback that should be called when the process shuts down.
-	registerShutdown(func() {
+	registerObjectStorageShutdown(func() errCode {
-		shutdownFS(storage)
+		return shutdownFS(storage)
 	})
 
 	// Initialize fs objects.

main.go

@@ -189,6 +189,10 @@ func main() {
 		defer profile.Start(profile.BlockProfile, profile.ProfilePath(profileDir)).Stop()
 	}
 
+	// Initialize and monitor shutdown signal
+	shutdownSignal = make(chan bool, 1)
+	monitorShutdownSignal()
+
 	// Run the app - exit on error.
 	app.RunAndExitOnError()
 }

object-common.go

@@ -17,11 +17,9 @@
 package main
 
 import (
-	"os"
 	"path/filepath"
 	"strings"
 	"sync"
-	"syscall"
 )
 
 const (
@@ -35,17 +33,6 @@ const (
 	bucketMetaPrefix = "buckets"
 )
 
-// Register callback functions that needs to be called when process shutsdown.
-// For now, SIGINT triggers the callbacks, in future controller can trigger
-// shutdown callbacks.
-func registerShutdown(callback func()) {
-	go func() {
-		trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
-		<-trapCh
-		callback()
-	}()
-}
-
 // isErrIgnored should we ignore this error?, takes a list of errors which can be ignored.
 func isErrIgnored(err error, ignoredErrs []error) bool {
 	for _, ignoredErr := range ignoredErrs {

server-main.go

@@ -275,6 +275,11 @@ func serverMain(c *cli.Context) {
 	// Prints the formatted startup message.
 	printStartupMessage(endPoints)
 
+	registerShutdown(func() errCode {
+		// apiServer.Stop()
+		return exitSuccess
+	})
+
 	// Start server.
 	// Configure TLS if certs are available.
 	if tls {
@@ -283,5 +288,6 @@ func serverMain(c *cli.Context) {
 		// Fallback to http.
 		err = apiServer.ListenAndServe()
 	}
+
 	fatalIf(err, "Failed to start minio server.")
 }

typed-errors.go

@@ -18,6 +18,14 @@ package main
 
 import "errors"
 
+// errCode represents the return status of shutdown functions
+type errCode int
+
+const (
+	exitFailure errCode = -1
+	exitSuccess errCode = 0
+)
+
 // errSyslogNotSupported - this message is only meaningful on windows
 var errSyslogNotSupported = errors.New("Syslog logger not supported on windows")
 

utils.go

@@ -20,7 +20,9 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"io"
+	"os"
 	"strings"
+	"syscall"
 )
 
 // xmlDecoder provide decoded value in xml.
@@ -73,3 +75,60 @@ func contains(stringList []string, element string) bool {
 	}
 	return false
 }
+
+// shutdownSignal - is the channel that receives any boolean when
+// we want broadcast the start of shutdown
+var shutdownSignal chan bool
+
+// shutdownCallbacks - is the list of function callbacks executed one by one
+// when a shutdown starts. A callback returns 0 for success and 1 for failure.
+// Failure is considered an emergency error that needs an immediate exit
+var shutdownCallbacks []func() errCode
+
+// shutdownObjectStorageCallbacks - contains the list of function callbacks that
+// need to be invoked when a shutdown starts. These callbacks will be called before
+// the general callback shutdowns
+var shutdownObjectStorageCallbacks []func() errCode
+
+// Register callback functions that need to be called when process terminates.
+func registerShutdown(callback func() errCode) {
+	shutdownCallbacks = append(shutdownCallbacks, callback)
+}
+
+// Register object storagecallback functions that need to be called when process terminates.
+func registerObjectStorageShutdown(callback func() errCode) {
+	shutdownObjectStorageCallbacks = append(shutdownObjectStorageCallbacks, callback)
+}
+
+// Start to monitor shutdownSignal to execute shutdown callbacks
+func monitorShutdownSignal() {
+	go func() {
+		// Monitor processus signal
+		trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
+		for {
+			select {
+			case <-trapCh:
+				// Start a graceful shutdown call
+				shutdownSignal <- true
+			case <-shutdownSignal:
+				// Call all callbacks and exit for emergency
+				for _, callback := range shutdownCallbacks {
+					exitCode := callback()
+					if exitCode != exitSuccess {
+						os.Exit(int(exitCode))
+					}
+
+				}
+				// Call all object storage shutdown callbacks and exit for emergency
+				for _, callback := range shutdownObjectStorageCallbacks {
+					exitCode := callback()
+					if exitCode != exitSuccess {
+						os.Exit(int(exitCode))
+					}
+
+				}
+				os.Exit(int(exitSuccess))
+			}
+		}
+	}()
+}

utils_test.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+// ShutdownCallback simulates a successful exit here, all registered
+// shutdown callbacks need to return exitSuccess for this test to succeed
+func TestShutdownCallback(t *testing.T) {
+	// Register two callbacks that return success
+	registerObjectStorageShutdown(func() errCode {
+		return exitSuccess
+	})
+	registerShutdown(func() errCode {
+		return exitSuccess
+	})
+
+	shutdownSignal = make(chan bool, 1)
+	shutdownSignal <- true
+	// Start executing callbacks and quit if everything is fine
+	monitorShutdownSignal()
+
+	// Infinite loop here simulates an infinite running program
+	for {
+	}
+}