Skip to content

Fix #10968: spawnProcess fails when RLIMIT_NOFILE exceeds int.max #10969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Poita wants to merge 2 commits into
base: master
Choose a base branch
from
+81 −52
Open

Fix #10968: spawnProcess fails when RLIMIT_NOFILE exceeds int.max #10969

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0269641
Fix process spawning failure when RLIMIT_NOFILE is unlimited
Poita Mar 5, 2026
8611dbb
Add unittest for process spawning with large RLIMIT_NOFILE
Poita Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 81 additions & 52 deletions std/process.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,21 +1046,16 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
if (getrlimit(RLIMIT_NOFILE, &r) != 0)
abortOnError(forkPipeOut, InternalError.getrlimit, .errno);

immutable maxDescriptors = cast(int) r.rlim_cur;
immutable long maxDescriptors = r.rlim_cur;

// Missing druntime declaration
pragma(mangle, "dirfd")
extern(C) nothrow @nogc int dirfd(DIR* dir);

DIR* dir = null;

// We read from /dev/fd or /proc/self/fd only if the limit is high enough
if (maxDescriptors > 128*1024)
{
// Try to open the directory /dev/fd or /proc/self/fd
dir = opendir("/dev/fd");
if (dir is null) dir = opendir("/proc/self/fd");
}
// Always try /dev/fd enumeration first — it's the most
// efficient approach and handles unlimited RLIMIT_NOFILE.
DIR* dir = opendir("/dev/fd");
if (dir is null) dir = opendir("/proc/self/fd");

// If we have a directory, close all file descriptors except stdin, stdout, and stderr
if (dir)
Expand All @@ -1085,52 +1080,52 @@ private Pid spawnProcessPosix(scope const(char[])[] args,
close(fd);
}
}
else
else if (maxDescriptors > 0 && maxDescriptors <= 128*1024)
{
// This is going to allocate 8 bytes for each possible file descriptor from lowfd to r.rlim_cur
if (maxDescriptors <= 128*1024)
// This is going to allocate 8 bytes for each possible file descriptor from lowfd to rlim_cur.
// NOTE: malloc() and getrlimit() are not on the POSIX async
// signal safe functions list, but practically this should
// not be a problem. Java VM and CPython also use malloc()
// in its own implementation via opendir().
import core.stdc.stdlib : malloc;
import core.sys.posix.poll : pollfd, poll, POLLNVAL;

immutable int maxToClose = cast(int)(maxDescriptors - lowfd);

// Call poll() to see which ones are actually open:
auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose);
if (pfds is null)
{
abortOnError(forkPipeOut, InternalError.malloc, .errno);
}

foreach (i; 0 .. maxToClose)
{
// NOTE: malloc() and getrlimit() are not on the POSIX async
// signal safe functions list, but practically this should
// not be a problem. Java VM and CPython also use malloc()
// in its own implementation via opendir().
import core.stdc.stdlib : malloc;
import core.sys.posix.poll : pollfd, poll, POLLNVAL;

immutable maxToClose = maxDescriptors - lowfd;

// Call poll() to see which ones are actually open:
auto pfds = cast(pollfd*) malloc(pollfd.sizeof * maxToClose);
if (pfds is null)
{
abortOnError(forkPipeOut, InternalError.malloc, .errno);
}

foreach (i; 0 .. maxToClose)
{
pfds[i].fd = i + lowfd;
pfds[i].events = 0;
pfds[i].revents = 0;
}

if (poll(pfds, maxToClose, 0) < 0)
// couldn't use poll, use the slow path.
goto LslowClose;

foreach (i; 0 .. maxToClose)
{
// POLLNVAL will be set if the file descriptor is invalid.
if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd);
}
pfds[i].fd = i + lowfd;
pfds[i].events = 0;
pfds[i].revents = 0;
}
else

if (poll(pfds, maxToClose, 0) < 0)
// couldn't use poll, use the slow path.
goto LslowClose;

foreach (i; 0 .. maxToClose)
{
// POLLNVAL will be set if the file descriptor is invalid.
if (!(pfds[i].revents & POLLNVAL)) close(pfds[i].fd);
}
}
else
{
LslowClose:
// Fall back to closing everything up to a sane limit.
// When rlim_cur is huge (e.g. unlimited), cap to avoid
// iterating over billions of file descriptors.
immutable long closeMax = maxDescriptors > 1_048_576 ? 1_048_576 : maxDescriptors;
foreach (i; lowfd .. cast(int) closeMax)
{
LslowClose:
// Fall back to closing everything.
foreach (i; lowfd .. maxDescriptors)
{
close(i);
}
close(i);
}
}
}
Expand Down Expand Up @@ -1800,6 +1795,40 @@ version (Posix) @system unittest
testFDs();
}

// Test that spawning a process works when RLIMIT_NOFILE is very large.
// Regression test: a cast(int) of rlim_cur caused overflow when the limit
// was unlimited (RLIM_INFINITY), making the fd-closing code attempt a
// massive malloc that would fail with "Cannot allocate memory".
version (Posix) @system unittest
{
import core.sys.posix.sys.resource : rlimit, getrlimit, setrlimit, RLIMIT_NOFILE;

// Save current limit
rlimit originalLimit;
if (getrlimit(RLIMIT_NOFILE, &originalLimit) != 0)
return; // Can't test if we can't get the limit

// Set RLIMIT_NOFILE to a value that overflows int (> int.max)
rlimit highLimit;
highLimit.rlim_cur = cast(ulong) int.max + 1;
highLimit.rlim_max = originalLimit.rlim_max;

// If we can't raise the limit (e.g. no permission), try with rlim_max
if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0)
{
highLimit.rlim_cur = originalLimit.rlim_max;
if (highLimit.rlim_cur <= int.max)
return; // Can't set a high enough limit to test the overflow
if (setrlimit(RLIMIT_NOFILE, &highLimit) != 0)
return;
}
scope(exit) setrlimit(RLIMIT_NOFILE, &originalLimit);

// This should not throw "Failed to allocate memory"
TestScript prog = "exit 0";
assert(execute(prog.path).status == 0);
}

@system unittest // Environment variables in spawnProcess().
{
// We really should use set /a on Windows, but Wine doesn't support it.
Expand Down
Loading