Move rlimit functions into sys package. (#3824)

This patch addresses below

* go build works for bsd family
* probe total RAM size for bsd family
* make unit testable functions

pkg/sys/rlimit-file_bsd.go

@@ -0,0 +1,40 @@
+// +build freebsd dragonfly
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import (
+	"syscall"
+)
+
+// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
+func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
+	var rlimit syscall.Rlimit
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err == nil {
+		curLimit = uint64(rlimit.Cur)
+		maxLimit = uint64(rlimit.Max)
+	}
+
+	return curLimit, maxLimit, err
+}
+
+// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
+func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
+	rlimit := syscall.Rlimit{Cur: int64(curLimit), Max: int64(curLimit)}
+	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
+}

pkg/sys/rlimit-file_nix.go

@@ -0,0 +1,38 @@
+// +build linux darwin openbsd netbsd
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "syscall"
+
+// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
+func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
+	var rlimit syscall.Rlimit
+	if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err == nil {
+		curLimit = rlimit.Cur
+		maxLimit = rlimit.Max
+	}
+
+	return curLimit, maxLimit, err
+}
+
+// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
+func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
+	rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
+	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
+}

pkg/sys/rlimit-file_test.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "testing"
+
+// Test get max open file limit.
+func TestGetMaxOpenFileLimit(t *testing.T) {
+	_, _, err := GetMaxOpenFileLimit()
+	if err != nil {
+		t.Errorf("expected: nil, got: %v", err)
+	}
+}
+
+// Test set open file limit
+func TestSetMaxOpenFileLimit(t *testing.T) {
+	curLimit, maxLimit, err := GetMaxOpenFileLimit()
+	if err != nil {
+		t.Fatalf("Unable to get max open file limit. %v", err)
+	}
+
+	err = SetMaxOpenFileLimit(curLimit, maxLimit)
+	if err != nil {
+		t.Errorf("expected: nil, got: %v", err)
+	}
+}

pkg/sys/rlimit-file_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+// GetMaxOpenFileLimit - returns maximum file descriptor number that can be opened by this process.
+func GetMaxOpenFileLimit() (curLimit, maxLimit uint64, err error) {
+	// Nothing to do for windows.
+	return curLimit, maxLimit, err
+}
+
+// SetMaxOpenFileLimit - sets maximum file descriptor number that can be opened by this process.
+func SetMaxOpenFileLimit(curLimit, maxLimit uint64) error {
+	// Nothing to do for windows.
+	return nil
+}

pkg/sys/rlimit-memory_bsd.go

@@ -0,0 +1,38 @@
+// +build freebsd dragonfly
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "syscall"
+
+// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
+func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
+	var rlimit syscall.Rlimit
+	if err = syscall.Getrlimit(syscall.RLIMIT_DATA, &rlimit); err == nil {
+		curLimit = uint64(rlimit.Cur)
+		maxLimit = uint64(rlimit.Max)
+	}
+
+	return curLimit, maxLimit, err
+}
+
+// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
+func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
+	rlimit := syscall.Rlimit{Cur: int64(curLimit), Max: int64(maxLimit)}
+	return syscall.Setrlimit(syscall.RLIMIT_DATA, &rlimit)
+}

pkg/sys/rlimit-memory_nix.go

@@ -0,0 +1,38 @@
+// +build linux darwin netbsd
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "syscall"
+
+// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
+func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
+	var rlimit syscall.Rlimit
+	if err = syscall.Getrlimit(syscall.RLIMIT_AS, &rlimit); err == nil {
+		curLimit = rlimit.Cur
+		maxLimit = rlimit.Max
+	}
+
+	return curLimit, maxLimit, err
+}
+
+// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
+func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
+	rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
+	return syscall.Setrlimit(syscall.RLIMIT_AS, &rlimit)
+}

pkg/sys/rlimit-memory_openbsd.go

@@ -0,0 +1,38 @@
+// +build openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "syscall"
+
+// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
+func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
+	var rlimit syscall.Rlimit
+	if err = syscall.Getrlimit(syscall.RLIMIT_DATA, &rlimit); err == nil {
+		curLimit = rlimit.Cur
+		maxLimit = rlimit.Max
+	}
+
+	return curLimit, maxLimit, err
+}
+
+// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
+func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
+	rlimit := syscall.Rlimit{Cur: curLimit, Max: maxLimit}
+	return syscall.Setrlimit(syscall.RLIMIT_DATA, &rlimit)
+}

pkg/sys/rlimit-memory_test.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+import "testing"
+
+// Test get max memory limit.
+func TestGetMaxMemoryLimit(t *testing.T) {
+	_, _, err := GetMaxMemoryLimit()
+	if err != nil {
+		t.Errorf("expected: nil, got: %v", err)
+	}
+}
+
+// Test set memory limit
+func TestSetMaxMemoryLimit(t *testing.T) {
+	curLimit, maxLimit, err := GetMaxMemoryLimit()
+	if err != nil {
+		t.Fatalf("Unable to get max memory limit. %v", err)
+	}
+
+	err = SetMaxMemoryLimit(curLimit, maxLimit)
+	if err != nil {
+		t.Errorf("expected: nil, got: %v", err)
+	}
+}

pkg/sys/rlimit-memory_windows.go

@@ -0,0 +1,31 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sys
+
+// GetMaxMemoryLimit - returns the maximum size of the process's virtual memory (address space) in bytes.
+func GetMaxMemoryLimit() (curLimit, maxLimit uint64, err error) {
+	// Nothing to do for windows.
+	return curLimit, maxLimit, err
+}
+
+// SetMaxMemoryLimit - sets the maximum size of the process's virtual memory (address space) in bytes.
+func SetMaxMemoryLimit(curLimit, maxLimit uint64) error {
+	// Nothing to do for windows.
+	return nil
+}

pkg/sys/stats.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,11 +16,6 @@
 
 package sys
 
-import "errors"
-
-// ErrNotImplemented - GetStats() is not implemented on bsds.
-var ErrNotImplemented = errors.New("not implemented")
-
 // Stats - system statistics.
 type Stats struct {
 	TotalRAM uint64 // Physical RAM size in bytes,

pkg/sys/stats_bsd.go

@@ -1,7 +1,7 @@
-// +build !linux,!windows,!darwin
+// +build openbsd netbsd freebsd dragonfly
 
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016,2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,28 @@
 
 package sys
 
-// GetStats - return system statistics for windows.
-func GetStats() (stats Stats, err error) {
-	return Stats{}, ErrNotImplemented
+import (
+	"encoding/binary"
+	"syscall"
+)
+
+func getHwPhysmem() (uint64, error) {
+	totalString, err := syscall.Sysctl("hw.physmem")
+	if err != nil {
+		return 0, err
+	}
+
+	// syscall.sysctl() helpfully assumes the result is a null-terminated string and
+	// removes the last byte of the result if it's 0 :/
+	totalString += "\x00"
+
+	total := uint64(binary.LittleEndian.Uint64([]byte(totalString)))
+
+	return total, nil
+}
+
+// GetStats - return system statistics for bsd.
+func GetStats() (stats Stats, err error) {
+	stats.TotalRAM, err = getHwPhysmem()
+	return stats, err
 }

pkg/sys/stats_darwin.go

@@ -1,7 +1,7 @@
 // +build darwin
 
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016,2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -38,14 +38,8 @@ func getHwMemsize() (uint64, error) {
 	return total, nil
 }
 
-// GetStats - return system statistics for windows.
+// GetStats - return system statistics for macOS.
 func GetStats() (stats Stats, err error) {
-	memSize, err := getHwMemsize()
-	if err != nil {
-		return Stats{}, err
-	}
-	stats = Stats{
-		TotalRAM: memSize,
-	}
-	return stats, nil
+	stats.TotalRAM, err = getHwMemsize()
+	return stats, err
 }

pkg/sys/stats_linux.go

@@ -1,7 +1,7 @@
 // +build linux
 
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016,2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,13 +22,10 @@ import "syscall"
 
 // GetStats - return system statistics.
 func GetStats() (stats Stats, err error) {
-	si := syscall.Sysinfo_t{}
-	err = syscall.Sysinfo(&si)
-	if err != nil {
-		return
+	var si syscall.Sysinfo_t
+	if err = syscall.Sysinfo(&si); err == nil {
+		stats.TotalRAM = uint64(si.Totalram)
 	}
-	stats = Stats{
-		TotalRAM: uint64(si.Totalram),
-	}
-	return stats, nil
+
+	return stats, err
 }

pkg/sys/stats_test.go

@@ -1,4 +1,18 @@
-// +build linux darwin windows
+/*
+ * Minio Cloud Storage, (C) 2016,2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package sys
 

pkg/sys/stats_windows.go

@@ -1,7 +1,7 @@
 // +build windows
 
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016,2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,12 +44,11 @@ type memoryStatusEx struct {
 func GetStats() (stats Stats, err error) {
 	var memInfo memoryStatusEx
 	memInfo.cbSize = uint32(unsafe.Sizeof(memInfo))
-	mem, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&memInfo)))
-	if mem == 0 {
-		return Stats{}, syscall.GetLastError()
+	if mem, _, _ := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&memInfo))); mem == 0 {
+		err = syscall.GetLastError()
+	} else {
+		stats.TotalRAM = memInfo.ullTotalPhys
 	}
-	stats = Stats{
-		TotalRAM: memInfo.ullTotalPhys,
-	}
-	return stats, nil
+
+	return stats, err
 }