MyFavMirrors/minio
1
0
Fork 0
mirror of https://github.com/minio/minio.git synced 2025-11-25 20:16:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Set the maximum open connections limit in PG and MySQL target configs (#10558)

Browse Source 
As the bulk/recursive delete will require multiple connections to open at an instance,
The default open connections limit will be reached which results in the following error

```FATAL:  sorry, too many clients already```

By setting the open connections to a reasonable value - `2`, We ensure that the max open connections
will not be exhausted and lie under bounds.

The queries are simple inserts/updates/deletes which is operational and sufficient with the
the maximum open connection limit is 2.

Fixes #10553

Allow user configuration for MaxOpenConnections
This commit is contained in:
Praveen raj Mani
2020-09-25 10:50:30 +05:30
committed by GitHub
parent 37a5d5d7a0
commit b880796aef
9 changed files with 198 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
pkg/event/target/mysql.go
View File

@@ -83,43 +83,46 @@ const (
// MySQL related constants
const (
MySQLFormat = "format"
MySQLDSNString = "dsn_string"
MySQLTable = "table"
MySQLHost = "host"
MySQLPort = "port"
MySQLUsername = "username"
MySQLPassword = "password"
MySQLDatabase = "database"
MySQLQueueLimit = "queue_limit"
MySQLQueueDir = "queue_dir"
MySQLFormat = "format"
MySQLDSNString = "dsn_string"
MySQLTable = "table"
MySQLHost = "host"
MySQLPort = "port"
MySQLUsername = "username"
MySQLPassword = "password"
MySQLDatabase = "database"
MySQLQueueLimit = "queue_limit"
MySQLQueueDir = "queue_dir"
MySQLMaxOpenConnections = "max_open_connections"
EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE"
EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT"
EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING"
EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE"
EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST"
EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT"
EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME"
EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD"
EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE"
EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE"
EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT"
EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING"
EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE"
EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST"
EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT"
EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME"
EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD"
EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE"
EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
EnvMySQLMaxOpenConnections = "MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS"
)
// MySQLArgs - MySQL target arguments.
type MySQLArgs struct {
Enable bool `json:"enable"`
Format string `json:"format"`
DSN string `json:"dsnString"`
Table string `json:"table"`
Host xnet.URL `json:"host"`
Port string `json:"port"`
User string `json:"user"`
Password string `json:"password"`
Database string `json:"database"`
QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"`
Enable bool `json:"enable"`
Format string `json:"format"`
DSN string `json:"dsnString"`
Table string `json:"table"`
Host xnet.URL `json:"host"`
Port string `json:"port"`
User string `json:"user"`
Password string `json:"password"`
Database string `json:"database"`
QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"`
MaxOpenConnections int `json:"maxOpenConnections"`
}
// Validate MySQLArgs fields
@@ -162,6 +165,10 @@ func (m MySQLArgs) Validate() error {
}
}
if m.MaxOpenConnections < 0 {
return errors.New("maxOpenConnections cannot be less than zero")
}
return nil
}
@@ -196,6 +203,10 @@ func (target *MySQLTarget) IsActive() (bool, error) {
return false, sErr
}
target.db = db
if target.args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(target.args.MaxOpenConnections)
}
}
if err := target.db.Ping(); err != nil {
if IsConnErr(err) {
@@ -384,6 +395,11 @@ func NewMySQLTarget(id string, args MySQLArgs, doneCh <-chan struct{}, loggerOnc
}
target.db = db
if args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(args.MaxOpenConnections)
}
var store Store
if args.QueueDir != "" {