ListObjects Metadata Caching (#10648)

Design: https://gist.github.com/klauspost/025c09b48ed4a1293c917cecfabdf21c

Gist of improvements:

* Cross-server caching and listing will use the same data across servers and requests.
* Lists can be arbitrarily resumed at a constant speed.
* Metadata for all files scanned is stored for streaming retrieval.
* The existing bloom filters controlled by the crawler is used for validating caches.
* Concurrent requests for the same data (or parts of it) will not spawn additional walkers.
* Listing a subdirectory of an existing recursive cache will use the cache.
* All listing operations are fully streamable so the number of objects in a bucket no 
  longer dictates the amount of memory.
* Listings can be handled by any server within the cluster.
* Caches are cleaned up when out of date or superseded by a more recent one.

cmd/metacache-marker.go

@@ -0,0 +1,70 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// markerTagVersion is the marker version.
+// Should not need to be updated unless a fundamental change is made to the marker format.
+const markerTagVersion = "v1"
+
+// parseMarker will parse a marker possibly encoded with encodeMarker
+func parseMarker(s string) (marker, uuid string) {
+	if !strings.Contains(s, "[minio_cache:"+markerTagVersion) {
+		return s, ""
+	}
+	start := strings.LastIndex(s, "[")
+	marker = s[:start]
+	end := strings.LastIndex(s, "]")
+	tag := strings.Trim(s[start:end], "[]")
+	tags := strings.Split(tag, ",")
+	for _, tag := range tags {
+		kv := strings.Split(tag, ":")
+		if len(kv) < 2 {
+			continue
+		}
+		switch kv[0] {
+		case "minio_cache":
+			if kv[1] != markerTagVersion {
+				break
+			}
+		case "id":
+			uuid = kv[1]
+		default:
+			// Ignore unknown
+		}
+	}
+	return
+}
+
+// encodeMarker will encode a uuid and return it as a marker.
+// uuid cannot contain '[', ':' or ','.
+func encodeMarker(marker, uuid string) string {
+	if uuid == "" {
+		return marker
+	}
+	if strings.ContainsAny(uuid, "[:,") {
+		logger.LogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", uuid))
+	}
+	return fmt.Sprintf("%s[minio_cache:%s,id:%s]", marker, markerTagVersion, uuid)
+}