mirror of
https://github.com/minio/minio.git
synced 2024-12-24 22:25:54 -05:00
Add initial cut of auto expiry of objects
This commit is contained in:
parent
c590acfc38
commit
179d2d7dac
6
main.go
6
main.go
@ -22,6 +22,7 @@ import (
|
|||||||
"os/user"
|
"os/user"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/minio/cli"
|
"github.com/minio/cli"
|
||||||
@ -34,8 +35,9 @@ type serverConfig struct {
|
|||||||
Anonymous bool // No signature turn off
|
Anonymous bool // No signature turn off
|
||||||
|
|
||||||
/// FS options
|
/// FS options
|
||||||
Path string // Path to export for cloud storage
|
Path string // Path to export for cloud storage
|
||||||
MinFreeDisk int64 // Minimum free disk space for filesystem
|
MinFreeDisk int64 // Minimum free disk space for filesystem
|
||||||
|
Expiry time.Duration // Set auto expiry for filesystem
|
||||||
|
|
||||||
// TLS service
|
// TLS service
|
||||||
TLS bool // TLS on when certs are specified
|
TLS bool // TLS on when certs are specified
|
||||||
|
41
pkg/fs/auto-expiry.go
Normal file
41
pkg/fs/auto-expiry.go
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AutoExpiryThread - auto expiry thread
|
||||||
|
func (fs Filesystem) AutoExpiryThread(expiry time.Duration) {
|
||||||
|
expireFiles := func(fp string, fl os.FileInfo, err error) error {
|
||||||
|
if fp == fs.path {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if fl.Mode().IsRegular() || fl.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||||
|
if time.Now().Sub(fl.ModTime()) > expiry {
|
||||||
|
if err := os.Remove(fp); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ErrDirNotEmpty
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(3 * time.Hour)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
// TODO - add a way to stop the timer thread
|
||||||
|
case <-ticker.C:
|
||||||
|
err := WalkUnsorted(fs.path, expireFiles)
|
||||||
|
if err != nil {
|
||||||
|
if !os.IsNotExist(err) && err != ErrDirNotEmpty {
|
||||||
|
ticker.Stop()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -59,6 +59,9 @@ func getNewCloudStorageAPI(conf serverConfig) CloudStorageAPI {
|
|||||||
|
|
||||||
fs.SetRootPath(conf.Path)
|
fs.SetRootPath(conf.Path)
|
||||||
fs.SetMinFreeDisk(conf.MinFreeDisk)
|
fs.SetMinFreeDisk(conf.MinFreeDisk)
|
||||||
|
if conf.Expiry > 0 {
|
||||||
|
go fs.AutoExpiryThread(conf.Expiry)
|
||||||
|
}
|
||||||
return CloudStorageAPI{
|
return CloudStorageAPI{
|
||||||
Filesystem: fs,
|
Filesystem: fs,
|
||||||
Anonymous: conf.Anonymous,
|
Anonymous: conf.Anonymous,
|
||||||
|
@ -26,6 +26,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/fatih/color"
|
"github.com/fatih/color"
|
||||||
"github.com/minio/cli"
|
"github.com/minio/cli"
|
||||||
@ -55,6 +56,10 @@ EXAMPLES:
|
|||||||
|
|
||||||
4. Start minio server with minimum free disk threshold to 5%
|
4. Start minio server with minimum free disk threshold to 5%
|
||||||
$ minio {{.Name}} min-free-disk 5% /home/shared/Pictures
|
$ minio {{.Name}} min-free-disk 5% /home/shared/Pictures
|
||||||
|
|
||||||
|
5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
|
||||||
|
$ minio {{.Name}} min-free-disk 15% expiry 1h /home/shared/Documents
|
||||||
|
|
||||||
`,
|
`,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -219,7 +224,7 @@ func checkServerSyntax(c *cli.Context) {
|
|||||||
if !c.Args().Present() || c.Args().First() == "help" {
|
if !c.Args().Present() || c.Args().First() == "help" {
|
||||||
cli.ShowCommandHelpAndExit(c, "server", 1)
|
cli.ShowCommandHelpAndExit(c, "server", 1)
|
||||||
}
|
}
|
||||||
if len(c.Args()) > 3 {
|
if len(c.Args()) > 5 {
|
||||||
fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
|
fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
|
||||||
}
|
}
|
||||||
path := strings.TrimSpace(c.Args().Last())
|
path := strings.TrimSpace(c.Args().Last())
|
||||||
@ -231,41 +236,66 @@ func checkServerSyntax(c *cli.Context) {
|
|||||||
func serverMain(c *cli.Context) {
|
func serverMain(c *cli.Context) {
|
||||||
checkServerSyntax(c)
|
checkServerSyntax(c)
|
||||||
|
|
||||||
err := fetchAuth()
|
perr := fetchAuth()
|
||||||
fatalIf(err.Trace(), "Failed to generate keys for minio.", nil)
|
fatalIf(perr.Trace(), "Failed to generate keys for minio.", nil)
|
||||||
|
|
||||||
path := strings.TrimSpace(c.Args().Last())
|
|
||||||
// Last argument is always path
|
|
||||||
if _, err := os.Stat(path); err != nil {
|
|
||||||
fatalIf(probe.NewError(err), "Unable to validate the path", nil)
|
|
||||||
}
|
|
||||||
certFile := c.GlobalString("cert")
|
certFile := c.GlobalString("cert")
|
||||||
keyFile := c.GlobalString("key")
|
keyFile := c.GlobalString("key")
|
||||||
if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
|
if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
|
||||||
fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
|
fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
var minFreeDisk int64
|
var minFreeDisk int64
|
||||||
// Only if args are greater than or equal to 2 verify if the proper variables are passed.
|
minFreeDiskSet := false
|
||||||
if len(c.Args()) >= 2 {
|
|
||||||
if c.Args().Get(0) == "min-free-disk" {
|
var expiration time.Duration
|
||||||
minFreeDisk, err = parsePercentToInt(c.Args().Get(1), 64)
|
expirationSet := false
|
||||||
fatalIf(err.Trace(c.Args().Get(2)), "Unable to parse minimum free disk parameter.", nil)
|
|
||||||
}
|
args := c.Args()
|
||||||
if c.Args().Get(0) != "min-free-disk" {
|
for len(args) >= 2 {
|
||||||
fatalIf(probe.NewError(errInvalidArgument), "Invalid arguments passed. ‘"+strings.Join(c.Args(), " ")+"’", nil)
|
switch args.First() {
|
||||||
|
case "min-free-disk":
|
||||||
|
if minFreeDiskSet {
|
||||||
|
fatalIf(probe.NewError(errInvalidArgument), "Minimum free disk should be set only once.", nil)
|
||||||
|
}
|
||||||
|
args = args.Tail()
|
||||||
|
var err *probe.Error
|
||||||
|
minFreeDisk, err = parsePercentToInt(args.First(), 64)
|
||||||
|
fatalIf(err.Trace(args.First()), "Invalid minium free disk size "+args.First()+" passed.", nil)
|
||||||
|
args = args.Tail()
|
||||||
|
minFreeDiskSet = true
|
||||||
|
case "expiry":
|
||||||
|
if expirationSet {
|
||||||
|
fatalIf(probe.NewError(errInvalidArgument), "Expiration should be set only once.", nil)
|
||||||
|
}
|
||||||
|
args = args.Tail()
|
||||||
|
var err error
|
||||||
|
expiration, err = time.ParseDuration(args.First())
|
||||||
|
fatalIf(probe.NewError(err), "Invalid expiration time "+args.First()+" passed.", nil)
|
||||||
|
args = args.Tail()
|
||||||
|
expirationSet = true
|
||||||
|
default:
|
||||||
|
cli.ShowCommandHelpAndExit(c, "server", 1) // last argument is exit code
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
path := strings.TrimSpace(c.Args().Last())
|
||||||
|
// Last argument is always path
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
fatalIf(probe.NewError(err), "Unable to validate the path", nil)
|
||||||
|
}
|
||||||
tls := (certFile != "" && keyFile != "")
|
tls := (certFile != "" && keyFile != "")
|
||||||
apiServerConfig := serverConfig{
|
apiServerConfig := serverConfig{
|
||||||
Address: c.GlobalString("address"),
|
Address: c.GlobalString("address"),
|
||||||
Anonymous: c.GlobalBool("anonymous"),
|
Anonymous: c.GlobalBool("anonymous"),
|
||||||
Path: path,
|
Path: path,
|
||||||
MinFreeDisk: minFreeDisk,
|
MinFreeDisk: minFreeDisk,
|
||||||
|
Expiry: expiration,
|
||||||
TLS: tls,
|
TLS: tls,
|
||||||
CertFile: certFile,
|
CertFile: certFile,
|
||||||
KeyFile: keyFile,
|
KeyFile: keyFile,
|
||||||
RateLimit: c.GlobalInt("ratelimit"),
|
RateLimit: c.GlobalInt("ratelimit"),
|
||||||
}
|
}
|
||||||
err = startServer(apiServerConfig)
|
perr = startServer(apiServerConfig)
|
||||||
errorIf(err.Trace(), "Failed to start the minio server.", nil)
|
errorIf(perr.Trace(), "Failed to start the minio server.", nil)
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user