From 026964165d5351d04a250c7dd15d258c885d2ffd Mon Sep 17 00:00:00 2001 From: Peter Alexander Date: Thu, 5 Mar 2026 08:30:32 +0000 Subject: [PATCH 1/2] Fix process spawning failure when RLIMIT_NOFILE is unlimited When RLIMIT_NOFILE is set to unlimited (RLIM_INFINITY), r.rlim_cur is a huge value (e.g. 2^63-1). The cast(int) on this value wraps to -1, which causes the /dev/fd fast path to be skipped (since -1 < 128K) and the poll() path to attempt a massive malloc that fails. This manifests as "Failed to allocate memory (Cannot allocate memory)" on any process spawn, making dub completely unusable on systems with unlimited file descriptor limits (common on macOS). Fix by: - Using long instead of cast(int) for maxDescriptors - Always trying /dev/fd enumeration first (it's the most efficient path and works regardless of the limit value) - Capping the slow close() fallback to 1M descriptors to avoid iterating over billions when the limit is huge Co-Authored-By: Claude Opus 4.6 --- std/process.d | 99 ++++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/std/process.d b/std/process.d index d359ca0bae4..c9b88e23d75 100644 --- a/std/process.d +++ b/std/process.d @@ -1046,21 +1046,16 @@ private Pid spawnProcessPosix(scope const(char[])[] args, if (getrlimit(RLIMIT_NOFILE, &r) != 0) abortOnError(forkPipeOut, InternalError.getrlimit, .errno); - immutable maxDescriptors = cast(int) r.rlim_cur; + immutable long maxDescriptors = r.rlim_cur; // Missing druntime declaration pragma(mangle, "dirfd") extern(C) nothrow @nogc int dirfd(DIR* dir); - DIR* dir = null; - - // We read from /dev/fd or /proc/self/fd only if the limit is high enough - if (maxDescriptors > 128*1024) - { - // Try to open the directory /dev/fd or /proc/self/fd - dir = opendir("/dev/fd"); - if (dir is null) dir = opendir("/proc/self/fd"); - } + // Always try /dev/fd enumeration first — it's the most + // efficient approach and handles unlimited RLIMIT_NOFILE. + DIR* dir = opendir("/dev/fd"); + if (dir is null) dir = opendir("/proc/self/fd"); // If we have a directory, close all file descriptors except stdin, stdout, and stderr if (dir) @@ -1085,52 +1080,52 @@ private Pid spawnProcessPosix(scope const(char[])[] args, close(fd); } } - else + else if (maxDescriptors > 0 && maxDescriptors <= 128*1024) { - // This is going to allocate 8 bytes for each possible file descriptor from lowfd to r.rlim_cur - if (maxDescriptors <= 128*1024) + // This is going to allocate 8 bytes for each possible file descriptor from lowfd to rlim_cur. + // NOTE: malloc() and getrlimit() are not on the POSIX async + // signal safe functions list, but practically this should + // not be a problem. Java VM and CPython also use malloc() + // in its own implementation via opendir(). + import core.stdc.stdlib : malloc; + import core.sys.posix.poll : pollfd, poll, POLLNVAL; + + immutable int maxToClose = cast(int)(maxDescriptors - lowfd); + + // Call poll() to see which ones are actually open: + auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose); + if (pfds is null) + { + abortOnError(forkPipeOut, InternalError.malloc, .errno); + } + + foreach (i; 0 .. maxToClose) { - // NOTE: malloc() and getrlimit() are not on the POSIX async - // signal safe functions list, but practically this should - // not be a problem. Java VM and CPython also use malloc() - // in its own implementation via opendir(). - import core.stdc.stdlib : malloc; - import core.sys.posix.poll : pollfd, poll, POLLNVAL; - - immutable maxToClose = maxDescriptors - lowfd; - - // Call poll() to see which ones are actually open: - auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose); - if (pfds is null) - { - abortOnError(forkPipeOut, InternalError.malloc, .errno); - } - - foreach (i; 0 .. maxToClose) - { - pfds[i].fd = i + lowfd; - pfds[i].events = 0; - pfds[i].revents = 0; - } - - if (poll(pfds, maxToClose, 0) < 0) - // couldn't use poll, use the slow path. - goto LslowClose; - - foreach (i; 0 .. maxToClose) - { - // POLLNVAL will be set if the file descriptor is invalid. - if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd); - } + pfds[i].fd = i + lowfd; + pfds[i].events = 0; + pfds[i].revents = 0; } - else + + if (poll(pfds, maxToClose, 0) < 0) + // couldn't use poll, use the slow path. + goto LslowClose; + + foreach (i; 0 .. maxToClose) + { + // POLLNVAL will be set if the file descriptor is invalid. + if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd); + } + } + else + { + LslowClose: + // Fall back to closing everything up to a sane limit. + // When rlim_cur is huge (e.g. unlimited), cap to avoid + // iterating over billions of file descriptors. + immutable long closeMax = maxDescriptors > 1_048_576 ? 1_048_576 : maxDescriptors; + foreach (i; lowfd .. cast(int) closeMax) { - LslowClose: - // Fall back to closing everything. - foreach (i; lowfd .. maxDescriptors) - { - close(i); - } + close(i); } } } From 8611dbb6d7728fef33e68fbf942b82b26cbba66e Mon Sep 17 00:00:00 2001 From: Peter Alexander Date: Thu, 5 Mar 2026 08:38:28 +0000 Subject: [PATCH 2/2] Add unittest for process spawning with large RLIMIT_NOFILE Regression test that sets RLIMIT_NOFILE above int.max and verifies process spawning still works. Without the previous fix, this triggers "Failed to allocate memory" due to integer overflow in the fd-closing code. Co-Authored-By: Claude Opus 4.6 --- std/process.d | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/std/process.d b/std/process.d index c9b88e23d75..ba1ce44ec87 100644 --- a/std/process.d +++ b/std/process.d @@ -1795,6 +1795,40 @@ version (Posix) @system unittest testFDs(); } +// Test that spawning a process works when RLIMIT_NOFILE is very large. +// Regression test: a cast(int) of rlim_cur caused overflow when the limit +// was unlimited (RLIM_INFINITY), making the fd-closing code attempt a +// massive malloc that would fail with "Cannot allocate memory". +version (Posix) @system unittest +{ + import core.sys.posix.sys.resource : rlimit, getrlimit, setrlimit, RLIMIT_NOFILE; + + // Save current limit + rlimit originalLimit; + if (getrlimit(RLIMIT_NOFILE, &originalLimit) != 0) + return; // Can't test if we can't get the limit + + // Set RLIMIT_NOFILE to a value that overflows int (> int.max) + rlimit highLimit; + highLimit.rlim_cur = cast(ulong) int.max + 1; + highLimit.rlim_max = originalLimit.rlim_max; + + // If we can't raise the limit (e.g. no permission), try with rlim_max + if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0) + { + highLimit.rlim_cur = originalLimit.rlim_max; + if (highLimit.rlim_cur <= int.max) + return; // Can't set a high enough limit to test the overflow + if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0) + return; + } + scope(exit) setrlimit(RLIMIT_NOFILE, &originalLimit); + + // This should not throw "Failed to allocate memory" + TestScript prog = "exit 0"; + assert(execute(prog.path).status == 0); +} + @system unittest // Environment variables in spawnProcess(). { // We really should use set /a on Windows, but Wine doesn't support it.