Implement HTTP POST based RPC (#5840)

Added support for new RPC support using HTTP POST. RPC's arguments and reply are Gob encoded and sent as HTTP request/response body. This patch also removes Go RPC based implementation.
2025-12-07 16:22:33 -05:00 · 2018-06-06 14:21:56 +05:30
parent 9d41051e91
commit 6a53dd1701
59 changed files with 5272 additions and 4169 deletions
--- a/cmd/storage-rpc-client.go
+++ b/cmd/storage-rpc-client.go
@@ -18,33 +18,33 @@ package cmd

 import (
 	"bytes"
+	"context"
+	"crypto/tls"
 	"io"
 	"net"
-	"net/rpc"
+	"net/url"
 	"path"
 	"strings"
+
+	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
 )

-type networkStorage struct {
-	rpcClient *AuthRPCClient
-	connected bool
-}
-
-const (
-	storageRPCPath = "/storage"
-)
-
-func isErrorNetworkDisconnect(err error) bool {
+func isNetworkDisconnectError(err error) bool {
 	if err == nil {
 		return false
 	}
-	if _, ok := err.(*net.OpError); ok {
-		return true
+
+	if uerr, isURLError := err.(*url.Error); isURLError {
+		if uerr.Timeout() {
+			return true
+		}
+
+		err = uerr.Err
 	}
-	if err == rpc.ErrShutdown {
-		return true
-	}
-	return false
+
+	_, isNetOpError := err.(*net.OpError)
+	return isNetOpError
 }

 // Converts rpc.ServerError to underlying error. This function is
@@ -55,7 +55,7 @@ func toStorageErr(err error) error {
 		return nil
 	}

-	if isErrorNetworkDisconnect(err) {
+	if isNetworkDisconnectError(err) {
 		return errDiskNotFound
 	}

@@ -100,162 +100,128 @@ func toStorageErr(err error) error {
 	return err
 }

-// Initialize new storage rpc client.
-func newStorageRPC(endpoint Endpoint) StorageAPI {
-	// Dial minio rpc storage http path.
-	rpcPath := path.Join(minioReservedBucketPath, storageRPCPath, endpoint.Path)
-	serverCred := globalServerConfig.GetCredential()
-
-	disk := &networkStorage{
-		rpcClient: newAuthRPCClient(authConfig{
-			accessKey:        serverCred.AccessKey,
-			secretKey:        serverCred.SecretKey,
-			serverAddr:       endpoint.Host,
-			serviceEndpoint:  rpcPath,
-			secureConn:       globalIsSSL,
-			serviceName:      "Storage",
-			disableReconnect: true,
-		}),
-	}
-	// Attempt a remote login.
-	disk.connected = disk.rpcClient.Login() == nil
-	return disk
+// StorageRPCClient - storage RPC client.
+type StorageRPCClient struct {
+	*RPCClient
+	connected bool
 }

 // Stringer provides a canonicalized representation of network device.
-func (n *networkStorage) String() string {
-	// Remove the storage RPC path prefix, internal paths are meaningless.
-	serviceEndpoint := strings.TrimPrefix(n.rpcClient.ServiceEndpoint(),
-		path.Join(minioReservedBucketPath, storageRPCPath))
-	// Check for the transport layer being used.
-	scheme := "http"
-	if n.rpcClient.config.secureConn {
-		scheme = "https"
-	}
-	// Finally construct the disk endpoint in http://<server>/<path> form.
-	return scheme + "://" + n.rpcClient.ServerAddr() + path.Join("/", serviceEndpoint)
+func (client *StorageRPCClient) String() string {
+	url := client.ServiceURL()
+	// Remove the storage RPC path prefix, internal paths are meaningless. why?
+	url.Path = strings.TrimPrefix(url.Path, storageServicePath)
+	return url.String()
 }

-func (n *networkStorage) Close() error {
-	n.connected = false
-	return toStorageErr(n.rpcClient.Close())
+// Close - closes underneath RPC client.
+func (client *StorageRPCClient) Close() error {
+	client.connected = false
+	return toStorageErr(client.RPCClient.Close())
 }

-func (n *networkStorage) IsOnline() bool {
-	return n.connected
+// IsOnline - returns whether RPC client failed to connect or not.
+func (client *StorageRPCClient) IsOnline() bool {
+	return client.connected
 }

-func (n *networkStorage) call(handler string, args interface {
-	SetAuthToken(string)
-	SetRPCAPIVersion(semVersion)
+func (client *StorageRPCClient) call(handler string, args interface {
+	SetAuthArgs(args AuthArgs)
 }, reply interface{}) error {
-	if !n.connected {
+	if !client.connected {
 		return errDiskNotFound
 	}
-	if err := n.rpcClient.Call(handler, args, reply); err != nil {
-		if isErrorNetworkDisconnect(err) {
-			n.connected = false
-		}
-		return toStorageErr(err)
+
+	err := client.Call(handler, args, reply)
+	if err == nil {
+		return nil
 	}
-	return nil
+
+	if isNetworkDisconnectError(err) {
+		client.connected = false
+	}
+
+	return toStorageErr(err)
 }

 // DiskInfo - fetch disk information for a remote disk.
-func (n *networkStorage) DiskInfo() (info DiskInfo, err error) {
-	args := AuthRPCArgs{}
-	if err = n.call("Storage.DiskInfoHandler", &args, &info); err != nil {
-		return DiskInfo{}, err
-	}
-	return info, nil
+func (client *StorageRPCClient) DiskInfo() (info DiskInfo, err error) {
+	err = client.call(storageServiceName+".DiskInfo", &AuthArgs{}, &info)
+	return info, err
 }

 // MakeVol - create a volume on a remote disk.
-func (n *networkStorage) MakeVol(volume string) (err error) {
-	reply := AuthRPCReply{}
-	args := GenericVolArgs{Vol: volume}
-	return n.call("Storage.MakeVolHandler", &args, &reply)
+func (client *StorageRPCClient) MakeVol(volume string) (err error) {
+	return client.call(storageServiceName+".MakeVol", &VolArgs{Vol: volume}, &VoidReply{})
 }

 // ListVols - List all volumes on a remote disk.
-func (n *networkStorage) ListVols() (vols []VolInfo, err error) {
-	ListVols := ListVolsReply{}
-	if err = n.call("Storage.ListVolsHandler", &AuthRPCArgs{}, &ListVols); err != nil {
-		return nil, err
-	}
-	return ListVols.Vols, nil
+func (client *StorageRPCClient) ListVols() ([]VolInfo, error) {
+	var reply []VolInfo
+	err := client.call(storageServiceName+".ListVols", &AuthArgs{}, &reply)
+	return reply, err
 }

 // StatVol - get volume info over the network.
-func (n *networkStorage) StatVol(volume string) (volInfo VolInfo, err error) {
-	args := GenericVolArgs{Vol: volume}
-	if err = n.call("Storage.StatVolHandler", &args, &volInfo); err != nil {
-		return VolInfo{}, err
-	}
-	return volInfo, nil
+func (client *StorageRPCClient) StatVol(volume string) (volInfo VolInfo, err error) {
+	err = client.call(storageServiceName+".StatVol", &VolArgs{Vol: volume}, &volInfo)
+	return volInfo, err
 }

 // DeleteVol - Deletes a volume over the network.
-func (n *networkStorage) DeleteVol(volume string) (err error) {
-	reply := AuthRPCReply{}
-	args := GenericVolArgs{Vol: volume}
-	return n.call("Storage.DeleteVolHandler", &args, &reply)
+func (client *StorageRPCClient) DeleteVol(volume string) (err error) {
+	return client.call(storageServiceName+".DeleteVol", &VolArgs{Vol: volume}, &VoidReply{})
 }

 // File operations.

-func (n *networkStorage) PrepareFile(volume, path string, length int64) (err error) {
-	reply := AuthRPCReply{}
-	return n.call("Storage.PrepareFileHandler", &PrepareFileArgs{
+// PrepareFile - calls PrepareFile RPC.
+func (client *StorageRPCClient) PrepareFile(volume, path string, length int64) (err error) {
+	args := PrepareFileArgs{
 		Vol:  volume,
 		Path: path,
 		Size: length,
-	}, &reply)
+	}
+	reply := VoidReply{}
+
+	return client.call(storageServiceName+".PrepareFile", &args, &reply)
 }

 // AppendFile - append file writes buffer to a remote network path.
-func (n *networkStorage) AppendFile(volume, path string, buffer []byte) (err error) {
-	reply := AuthRPCReply{}
-	return n.call("Storage.AppendFileHandler", &AppendFileArgs{
+func (client *StorageRPCClient) AppendFile(volume, path string, buffer []byte) (err error) {
+	args := AppendFileArgs{
 		Vol:    volume,
 		Path:   path,
 		Buffer: buffer,
-	}, &reply)
+	}
+	reply := VoidReply{}
+
+	return client.call(storageServiceName+".AppendFile", &args, &reply)
 }

 // StatFile - get latest Stat information for a file at path.
-func (n *networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err error) {
-	if err = n.call("Storage.StatFileHandler", &StatFileArgs{
-		Vol:  volume,
-		Path: path,
-	}, &fileInfo); err != nil {
-		return FileInfo{}, err
-	}
-	return fileInfo, nil
+func (client *StorageRPCClient) StatFile(volume, path string) (fileInfo FileInfo, err error) {
+	err = client.call(storageServiceName+".StatFile", &StatFileArgs{Vol: volume, Path: path}, &fileInfo)
+	return fileInfo, err
 }

 // ReadAll - reads entire contents of the file at path until EOF, returns the
 // contents in a byte slice. Returns buf == nil if err != nil.
 // This API is meant to be used on files which have small memory footprint, do
 // not use this on large files as it would cause server to crash.
-func (n *networkStorage) ReadAll(volume, path string) (buf []byte, err error) {
-	if err = n.call("Storage.ReadAllHandler", &ReadAllArgs{
-		Vol:  volume,
-		Path: path,
-	}, &buf); err != nil {
-		return nil, err
-	}
-	return buf, nil
+func (client *StorageRPCClient) ReadAll(volume, path string) (buf []byte, err error) {
+	err = client.call(storageServiceName+".ReadAll", &ReadAllArgs{Vol: volume, Path: path}, &buf)
+	return buf, err
 }

 // ReadFile - reads a file at remote path and fills the buffer.
-func (n *networkStorage) ReadFile(volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (m int64, err error) {
+func (client *StorageRPCClient) ReadFile(volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (m int64, err error) {
+	// Recover from any panic and return error.
 	defer func() {
 		if r := recover(); r != nil {
-			// Recover any panic from allocation, and return error.
 			err = bytes.ErrTooLarge
 		}
-	}() // Do not crash the server.
+	}()

 	args := ReadFileArgs{
 		Vol:      volume,
@@ -269,46 +235,90 @@ func (n *networkStorage) ReadFile(volume string, path string, offset int64, buff
 		args.ExpectedHash = verifier.sum
 		args.Verified = verifier.IsVerified()
 	}
+	var reply []byte

-	var result []byte
-	err = n.call("Storage.ReadFileHandler", &args, &result)
+	err = client.call(storageServiceName+".ReadFile", &args, &reply)

-	// Copy results to buffer.
-	copy(buffer, result)
+	// Copy reply to buffer.
+	copy(buffer, reply)

 	// Return length of result, err if any.
-	return int64(len(result)), err
+	return int64(len(reply)), err
 }

 // ListDir - list all entries at prefix.
-func (n *networkStorage) ListDir(volume, path string, count int) (entries []string, err error) {
-	if err = n.call("Storage.ListDirHandler", &ListDirArgs{
-		Vol:   volume,
-		Path:  path,
-		Count: count,
-	}, &entries); err != nil {
-		return nil, err
-	}
-	// Return successfully unmarshalled results.
-	return entries, nil
+func (client *StorageRPCClient) ListDir(volume, path string, count int) (entries []string, err error) {
+	err = client.call(storageServiceName+".ListDir", &ListDirArgs{Vol: volume, Path: path, Count: count}, &entries)
+	return entries, err
 }

 // DeleteFile - Delete a file at path.
-func (n *networkStorage) DeleteFile(volume, path string) (err error) {
-	reply := AuthRPCReply{}
-	return n.call("Storage.DeleteFileHandler", &DeleteFileArgs{
+func (client *StorageRPCClient) DeleteFile(volume, path string) (err error) {
+	args := DeleteFileArgs{
 		Vol:  volume,
 		Path: path,
-	}, &reply)
+	}
+	reply := VoidReply{}
+
+	return client.call(storageServiceName+".DeleteFile", &args, &reply)
 }

 // RenameFile - rename a remote file from source to destination.
-func (n *networkStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
-	reply := AuthRPCReply{}
-	return n.call("Storage.RenameFileHandler", &RenameFileArgs{
+func (client *StorageRPCClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	args := RenameFileArgs{
 		SrcVol:  srcVolume,
 		SrcPath: srcPath,
 		DstVol:  dstVolume,
 		DstPath: dstPath,
-	}, &reply)
+	}
+	reply := VoidReply{}
+
+	return client.call(storageServiceName+".RenameFile", &args, &reply)
+}
+
+// NewStorageRPCClient - returns new storage RPC client.
+func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClient, error) {
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serviceURL := &xnet.URL{
+		Scheme: scheme,
+		Host:   host.String(),
+		Path:   path.Join(storageServicePath, endpointPath),
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: host.Name,
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	rpcClient, err := NewRPCClient(
+		RPCClientArgs{
+			NewAuthTokenFunc: newAuthToken,
+			RPCVersion:       globalRPCAPIVersion,
+			ServiceName:      storageServiceName,
+			ServiceURL:       serviceURL,
+			TLSConfig:        tlsConfig,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &StorageRPCClient{RPCClient: rpcClient}, nil
+}
+
+// Initialize new storage rpc client.
+func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
+	host, err := xnet.ParseHost(endpoint.Host)
+	logger.CriticalIf(context.Background(), err)
+	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
+	logger.CriticalIf(context.Background(), err)
+	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
+	return rpcClient
 }