Independent Multipart Uploads (#15346)

Do completely independent multipart uploads.

In distributed mode, a lock was held to merge each multipart 
upload as it was added. This lock was highly contested and 
retries are expensive (timewise) in distributed mode.

Instead, each part adds its metadata information uniquely. 
This eliminates the per object lock required for each to merge.
The metadata is read back and merged by "CompleteMultipartUpload" 
without locks when constructing final object.

Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/storage-rest-server.go

@@ -1244,6 +1244,47 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+// ReadMultiple returns multiple files
+func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	rw := streamHTTPResponse(w)
+	defer func() {
+		if r := recover(); r != nil {
+			debug.PrintStack()
+			rw.CloseWithError(fmt.Errorf("panic: %v", r))
+		}
+	}()
+
+	var req ReadMultipleReq
+	mr := msgpNewReader(r.Body)
+	err := req.DecodeMsg(mr)
+	if err != nil {
+		rw.CloseWithError(err)
+		return
+	}
+
+	mw := msgp.NewWriter(rw)
+	responses := make(chan ReadMultipleResp, len(req.Files))
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for resp := range responses {
+			err := resp.EncodeMsg(mw)
+			if err != nil {
+				rw.CloseWithError(err)
+				return
+			}
+			mw.Flush()
+		}
+	}()
+	err = s.storage.ReadMultiple(r.Context(), req, responses)
+	wg.Wait()
+	rw.CloseWithError(err)
+}
+
 // registerStorageRPCRouter - register storage rpc router.
 func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools) {
 	storageDisks := make([][]*xlStorage, len(endpointServerPools))
@@ -1315,6 +1356,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFileHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(httpTraceHdrs(server.ReadMultiple))
 		}
 	}
 }