add option for O_SYNC writes for standalone FS backend (#9581)

This commit is contained in:
Krishna Srinivas 2020-05-12 19:24:59 -07:00 committed by GitHub
parent c045ae15e7
commit 94f1a1dea3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 29 additions and 7 deletions

View File

@ -186,6 +186,11 @@ func handleCommonEnvVars() {
logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
}
globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
if err != nil {
logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
}
domains := env.Get(config.EnvDomain, "")
if len(domains) != 0 {
for _, domainName := range strings.Split(domains, config.ValueSeparator) {

View File

@ -32,6 +32,7 @@ const (
EnvRegionName = "MINIO_REGION_NAME"
EnvPublicIPs = "MINIO_PUBLIC_IPS"
EnvEndpoints = "MINIO_ENDPOINTS"
EnvFSOSync = "MINIO_FS_OSYNC"
// API sub-system
EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX"

View File

@ -24,6 +24,12 @@ var (
"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
)
ErrInvalidFSOSyncValue = newErrFn(
"Invalid O_SYNC value",
"Please check the passed value",
"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
)
ErrInvalidDomainValue = newErrFn(
"Invalid domain value",
"Please check the passed value",

View File

@ -321,7 +321,11 @@ func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []
return 0, err
}
writer, err := lock.Open(filePath, os.O_CREATE|os.O_WRONLY, 0666)
flags := os.O_CREATE | os.O_WRONLY
if globalFSOSync {
flags = flags | os.O_SYNC
}
writer, err := lock.Open(filePath, flags, 0666)
if err != nil {
return 0, osErrToFSFileErr(err)
}

View File

@ -112,7 +112,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
}
partPath := pathJoin(uploadIDDir, entry)
err = mioutil.AppendFile(file.filePath, partPath)
err = mioutil.AppendFile(file.filePath, partPath, globalFSOSync)
if err != nil {
reqInfo := logger.GetReqInfo(ctx).AppendTags("partPath", partPath)
reqInfo.AppendTags("filepath", file.filePath)
@ -638,7 +638,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
}
for _, part := range parts {
partPath := getPartFile(entries, part.PartNumber, part.ETag)
if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath)); err != nil {
if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath), globalFSOSync); err != nil {
logger.LogIf(ctx, err)
return oi, toObjectErr(err)
}

View File

@ -280,6 +280,8 @@ var (
// fix the system.
globalSafeMode bool
// If writes to FS backend should be O_SYNC.
globalFSOSync bool
// Add new variable global values here.
)

View File

@ -24,8 +24,12 @@ import (
)
// AppendFile - appends the file "src" to the file "dst"
func AppendFile(dst string, src string) error {
appendFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
func AppendFile(dst string, src string, osync bool) error {
flags := os.O_WRONLY | os.O_APPEND | os.O_CREATE
if osync {
flags = flags | os.O_SYNC
}
appendFile, err := os.OpenFile(dst, flags, 0666)
if err != nil {
return err
}

View File

@ -24,7 +24,7 @@ import (
)
// AppendFile - appends the file "src" to the file "dst"
func AppendFile(dst string, src string) error {
func AppendFile(dst string, src string, osync bool) error {
appendFile, err := lock.Open(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
if err != nil {
return err

View File

@ -61,7 +61,7 @@ func TestAppendFile(t *testing.T) {
f.WriteString("bbbbbbbbbb")
f.Close()
if err = AppendFile(name1, name2); err != nil {
if err = AppendFile(name1, name2, false); err != nil {
t.Error(err)
}