cache: allow ellipse style entries for MINIO_CACHE_DRIVES (#6088)

Fixes #5863

cmd/disk-cache-config.go

@@ -19,6 +19,9 @@ package cmd
 import (
 	"encoding/json"
 	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/ellipses"
 )
 
 // CacheConfig represents cache config settings
@@ -52,12 +55,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 
 // Parses given cacheDrivesEnv and returns a list of cache drives.
 func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
 	for _, d := range drives {
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
+		}
+	}
+
+	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
-	return drives, nil
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
 }
 
 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.

cmd/disk-cache-config_test.go

@@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
 	} else {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
 		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))

docs/config/README.md

@@ -110,6 +110,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
 |``exclude`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache |
 |``expiry`` | _int_ | Days to cache expiry |
+|``maxuse`` | _int_ | Percentage of disk available to cache |
 
 #### Notify
 |Field|Type|Description|

docs/config/config.sample.json

@@ -1,11 +1,12 @@
 {
-    "version": "24",
+    "version": "26",
     "credential": {
         "accessKey": "USWUXHGYZQYFYFFIT3RE",
         "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
     },
     "region": "us-east-1",
     "browser": "on",
+    "worm": "off",
     "domain": "",
     "storageclass": {
         "standard": "",
@@ -14,11 +15,12 @@
     "cache": {
         "drives": [],
         "expiry": 90,
-        "exclude": []
+        "exclude": [],
+        "maxuse": 80
     },
     "usage": {
         "interval": "3h"
-    }
+    },
     "notify": {
         "amqp": {
             "1": {
@@ -124,3 +126,4 @@
         }
     }
 }
+

docs/disk-caching/DESIGN.md

@@ -16,10 +16,10 @@ minio server -h
 ...
 ...
 
-  7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/drive3',
+  7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
      exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
      with expiry upto 40 days.
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
+     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}"
      $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
      $ export MINIO_CACHE_EXPIRY=40
      $ export MINIO_CACHE_MAXUSE=80

docs/disk-caching/README.md

@@ -17,14 +17,15 @@ Disk caching can be enabled by updating the `cache` config settings for Minio se
 "cache": {
 	"drives": ["/mnt/drive1", "/mnt/drive2", "/mnt/drive3"],
 	"expiry": 90,
-	"exclude": ["*.pdf","mybucket/*"]
+	"exclude": ["*.pdf","mybucket/*"],
+	"maxuse" : 70,
 },
 ```
 
-The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` and `/mnt/drive3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup.
+The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.
 
 ```bash
-export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
+export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"
 export MINIO_CACHE_EXPIRY=90
 export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
 export MINIO_CACHE_MAXUSE=80