From 70e1cbda21b2e41069f0167e64bb1f7c55799cbd Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Mon, 17 Jan 2022 08:34:14 -0800
Subject: [PATCH] allow disabling O_DIRECT in certain environments for reads
 (#14115)

repeated reads on single large objects in HPC like
workloads, need the following option to disable
O_DIRECT for a more effective usage of the kernel
page-cache.

However this optional should be used in very specific
situations only, and shouldn't be enabled on all
servers.

NVMe servers benefit always from keeping O_DIRECT on.
---
 cmd/handler-api.go          |  9 +++++++++
 cmd/xl-storage.go           |  2 +-
 internal/config/api/api.go  | 10 ++++++++++
 internal/config/api/help.go |  6 ++++++
 4 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/cmd/handler-api.go b/cmd/handler-api.go
index 8839cf54a..29b41c2a7 100644
--- a/cmd/handler-api.go
+++ b/cmd/handler-api.go
@@ -49,6 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
+	disableODirect              bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -150,6 +151,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
+	t.disableODirect = cfg.DisableODirect
+}
+
+func (t *apiConfig) isDisableODirect() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.disableODirect
 }
 
 func (t *apiConfig) getListQuorum() int {
diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go
index 34a1e5d9d..196edc9fe 100644
--- a/cmd/xl-storage.go
+++ b/cmd/xl-storage.go
@@ -1652,7 +1652,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	}
 
 	alignment := offset%xioutil.DirectioAlignSize == 0
-	if !alignment {
+	if !alignment || globalAPIConfig.isDisableODirect() {
 		if err = disk.DisableDirectIO(file); err != nil {
 			file.Close()
 			return nil, err
diff --git a/internal/config/api/api.go b/internal/config/api/api.go
index bfb0cb181..372e8a507 100644
--- a/internal/config/api/api.go
+++ b/internal/config/api/api.go
@@ -43,6 +43,7 @@ const (
 	apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
 	apiStaleUploadsExpiry          = "stale_uploads_expiry"
 	apiDeleteCleanupInterval       = "delete_cleanup_interval"
+	apiDisableODirect              = "disable_odirect"
 
 	EnvAPIRequestsMax              = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline         = "MINIO_API_REQUESTS_DEADLINE"
@@ -59,6 +60,7 @@ const (
 	EnvAPIStaleUploadsExpiry          = "MINIO_API_STALE_UPLOADS_EXPIRY"
 	EnvAPIDeleteCleanupInterval       = "MINIO_API_DELETE_CLEANUP_INTERVAL"
 	EnvDeleteCleanupInterval          = "MINIO_DELETE_CLEANUP_INTERVAL"
+	EnvAPIDisableODirect              = "MINIO_API_DISABLE_ODIRECT"
 )
 
 // Deprecated key and ENVs
@@ -118,6 +120,10 @@ var (
 			Key:   apiDeleteCleanupInterval,
 			Value: "5m",
 		},
+		config.KV{
+			Key:   apiDisableODirect,
+			Value: "off",
+		},
 	}
 )
 
@@ -135,6 +141,7 @@ type Config struct {
 	StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
 	StaleUploadsExpiry          time.Duration `json:"stale_uploads_expiry"`
 	DeleteCleanupInterval       time.Duration `json:"delete_cleanup_interval"`
+	DisableODirect              bool          `json:"disable_odirect"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -254,6 +261,8 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn
+
 	return Config{
 		RequestsMax:                 requestsMax,
 		RequestsDeadline:            requestsDeadline,
@@ -267,5 +276,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		StaleUploadsCleanupInterval: staleUploadsCleanupInterval,
 		StaleUploadsExpiry:          staleUploadsExpiry,
 		DeleteCleanupInterval:       deleteCleanupInterval,
+		DisableODirect:              disableODirect,
 	}, nil
 }
diff --git a/internal/config/api/help.go b/internal/config/api/help.go
index 61ed49dc9..01d13a67d 100644
--- a/internal/config/api/help.go
+++ b/internal/config/api/help.go
@@ -94,5 +94,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiDisableODirect,
+			Description: "set to disable O_DIRECT for reads under special conditions. NOTE: it is not recommended to disable O_DIRECT without prior testing.",
+			Optional:    true,
+			Type:        "boolean",
+		},
 	}
 )