Update file descriptor limits for macOS (#892)

* Introduces proper file handle limits for darwin based OSes (macOS/OSX)
for working with Go 1.12

Author: prd-fox

cmd/utils/flags.go

@@ -882,10 +882,11 @@ func makeDatabaseHandles() int {
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if err := fdlimit.Raise(uint64(limit)); err != nil {
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
-	return limit / 2 // Leave half for networking and other stuff
+	return int(raised / 2) // Leave half for networking and other stuff
 }
 
 // MakeAddress converts an account specified directly as a hex encoded string or

common/fdlimit/fdlimit_darwin.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "syscall"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 10240
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	// Retrieve the maximum allowed by dynamic OS limits
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Cap it to OPEN_MAX (10240) because macos is a special snowflake
+	if limit.Max > hardlimit {
+		limit.Max = hardlimit
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_freebsd.go

@@ -26,21 +26,24 @@ import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
 	if limit.Cur > int64(max) {
 		limit.Cur = int64(max)
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
+	}
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
 	}
-	return nil
+	return uint64(limit.Cur), nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_test.go

@@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
 	if limit, err := Current(); err != nil || limit <= 0 {
 		t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
 	}
-	if err := Raise(uint64(target)); err != nil {
+	if _, err := Raise(uint64(target)); err != nil {
 		t.Fatalf("failed to raise file allowance")
 	}
 	if limit, err := Current(); err != nil || limit < target {

common/fdlimit/fdlimit_unix.go

@@ -14,29 +14,34 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin netbsd openbsd solaris
+// +build linux netbsd openbsd solaris
 
 package fdlimit
 
 import "syscall"
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
 	// Get the current limit
 	var limit syscall.Rlimit
 	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
 	}
 	// Try to update the limit to the max allowance
 	limit.Cur = limit.Max
 	if limit.Cur > max {
 		limit.Cur = max
 	}
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
-		return err
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
 	}
-	return nil
+	return limit.Cur, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this

common/fdlimit/fdlimit_windows.go

@@ -16,28 +16,31 @@
 
 package fdlimit
 
-import "errors"
+import "fmt"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process
 // to the maximum hard-limit allowed by the OS.
-func Raise(max uint64) error {
+func Raise(max uint64) (uint64, error) {
 	// This method is NOP by design:
 	//  * Linux/Darwin counterparts need to manually increase per process limits
 	//  * On Windows Go uses the CreateFile API, which is limited to 16K files, non
 	//    changeable from within a running process
 	// This way we can always "request" raising the limits, which will either have
 	// or not have effect based on the platform we're running on.
-	if max > 16384 {
-		return errors.New("file descriptor limit (16384) reached")
+	if max > hardlimit {
+		return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
 	}
-	return nil
+	return max, nil
 }
 
 // Current retrieves the number of file descriptors allowed to be opened by this
 // process.
 func Current() (int, error) {
 	// Please see Raise for the reason why we use hard coded 16K as the limit
-	return 16384, nil
+	return hardlimit, nil
 }
 
 // Maximum retrieves the maximum number of file descriptors this process is

core/tx_pool_test.go

@@ -139,7 +139,7 @@ func validateEvents(events chan NewTxsEvent, count int) error {
 		case ev := <-events:
 			received = append(received, ev.Txs...)
 		case <-time.After(time.Second):
-			return fmt.Errorf("event #%d not fired", received)
+			return fmt.Errorf("event #%d not fired", len(received))
 		}
 	}
 	if len(received) > count {

crypto/bn256/cloudflare/main_test.go

@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
 			t.Error(err)
 			continue
 		}
-		t.Logf("%d: %x\n", n, g2.Marshal())
+		t.Logf("%v: %x\n", n, g2.Marshal())
 	}
 }
 

crypto/bn256/google/main_test.go

@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
 			t.Error(err)
 			continue
 		}
-		t.Logf("%d: %x\n", n, g2.Marshal())
+		t.Logf("%v: %x\n", n, g2.Marshal())
 	}
 }