Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions Lib/multiprocessing/forkserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,18 +162,25 @@ def ensure_running(self):
self._forkserver_alive_fd = None
self._forkserver_pid = None

cmd = ('from multiprocessing.forkserver import main; ' +
'main(%d, %d, %r, **%r)')
# gh-144503: sys_argv is passed as real argv elements after the
# ``-c cmd`` rather than repr'd into main_kws so that a large
# parent sys.argv cannot push the single ``-c`` command string
# over the OS per-argument length limit (MAX_ARG_STRLEN on Linux).
# The child sees them as sys.argv[1:].
cmd = ('import sys; '
'from multiprocessing.forkserver import main; '
'main(%d, %d, %r, sys_argv=sys.argv[1:], **%r)')

main_kws = {}
sys_argv = None
if self._preload_modules:
data = spawn.get_preparation_data('ignore')
if 'sys_path' in data:
main_kws['sys_path'] = data['sys_path']
if 'init_main_from_path' in data:
main_kws['main_path'] = data['init_main_from_path']
if 'sys_argv' in data:
main_kws['sys_argv'] = data['sys_argv']
sys_argv = data['sys_argv']
if self._preload_on_error != 'ignore':
main_kws['on_error'] = self._preload_on_error

Expand All @@ -197,6 +204,8 @@ def ensure_running(self):
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd]
if sys_argv is not None:
args += sys_argv
pid = util.spawnv_passfds(exe, args, fds_to_pass)
except:
os.close(alive_w)
Expand Down
71 changes: 70 additions & 1 deletion Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,10 +207,38 @@ def spawn_check_wrapper(*args, **kwargs):
return decorator


def only_run_in_forkserver_testsuite(reason):
"""Returns a decorator: raises SkipTest unless fork is supported
and the current start method is forkserver.

Combines @support.requires_fork() with the single-run semantics of
only_run_in_spawn_testsuite(), but uses the forkserver testsuite as
the single-run target. Appropriate for tests that exercise
os.fork() directly (raw fork or mp.set_start_method("fork") in a
subprocess) and don't vary by start method, since forkserver is
only available on platforms that support fork.
"""

def decorator(test_item):

@functools.wraps(test_item)
def forkserver_check_wrapper(*args, **kwargs):
if not support.has_fork_support:
raise unittest.SkipTest("requires working os.fork()")
if (start_method := multiprocessing.get_start_method()) != "forkserver":
raise unittest.SkipTest(
f"{start_method=}, not 'forkserver'; {reason}")
return test_item(*args, **kwargs)

return forkserver_check_wrapper

return decorator


class TestInternalDecorators(unittest.TestCase):
"""Logic within a test suite that could errantly skip tests? Test it!"""

@unittest.skipIf(sys.platform == "win32", "test requires that fork exists.")
@support.requires_fork()
def test_only_run_in_spawn_testsuite(self):
if multiprocessing.get_start_method() != "spawn":
raise unittest.SkipTest("only run in test_multiprocessing_spawn.")
Expand All @@ -234,6 +262,30 @@ def return_four_if_spawn():
finally:
multiprocessing.set_start_method(orig_start_method, force=True)

@support.requires_fork()
def test_only_run_in_forkserver_testsuite(self):
if multiprocessing.get_start_method() != "forkserver":
raise unittest.SkipTest("only run in test_multiprocessing_forkserver.")

try:
@only_run_in_forkserver_testsuite("testing this decorator")
def return_four_if_forkserver():
return 4
except Exception as err:
self.fail(f"expected decorated `def` not to raise; caught {err}")

orig_start_method = multiprocessing.get_start_method(allow_none=True)
try:
multiprocessing.set_start_method("forkserver", force=True)
self.assertEqual(return_four_if_forkserver(), 4)
multiprocessing.set_start_method("spawn", force=True)
with self.assertRaises(unittest.SkipTest) as ctx:
return_four_if_forkserver()
self.assertIn("testing this decorator", str(ctx.exception))
self.assertIn("start_method=", str(ctx.exception))
finally:
multiprocessing.set_start_method(orig_start_method, force=True)


#
# Creates a wrapper for a function which records the time it takes to finish
Expand Down Expand Up @@ -7133,6 +7185,23 @@ def test_preload_main_sys_argv(self):
'',
])

@only_run_in_forkserver_testsuite("forkserver specific test.")
def test_preload_main_large_sys_argv(self):
# gh-144503: a very large parent sys.argv must not prevent the
# forkserver from starting (it previously overflowed the OS
# per-argument length limit when repr'd into the -c command string).
name = os.path.join(os.path.dirname(__file__),
'mp_preload_large_sysargv.py')
_, out, err = test.support.script_helper.assert_python_ok(name)
self.assertEqual(err, b'')

out = out.decode().split("\n")
self.assertEqual(out, [
'preload:5002:sentinel',
'worker:5002:sentinel',
'',
])

#
# Mixins
#
Expand Down
30 changes: 30 additions & 0 deletions Lib/test/mp_preload_large_sysargv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# gh-144503: Test that the forkserver can start when the parent process has
# a very large sys.argv. Prior to the fix, sys.argv was repr'd into the
# forkserver ``-c`` command string which could exceed the OS limit on the
# length of a single argv element (MAX_ARG_STRLEN on Linux, ~128 KiB),
# causing posix_spawn to fail and the parent to see a BrokenPipeError.

import multiprocessing
import sys

EXPECTED_LEN = 5002 # argv[0] + 5000 padding entries + sentinel


def fun():
print(f"worker:{len(sys.argv)}:{sys.argv[-1]}")


if __name__ == "__main__":
# Inflate sys.argv well past 128 KiB before the forkserver is started.
sys.argv[1:] = ["x" * 50] * 5000 + ["sentinel"]
assert len(sys.argv) == EXPECTED_LEN

ctx = multiprocessing.get_context("forkserver")
p = ctx.Process(target=fun)
p.start()
p.join()
sys.exit(p.exitcode)
else:
# This branch runs when the forkserver preloads this module as
# __mp_main__; confirm the large argv was propagated intact.
print(f"preload:{len(sys.argv)}:{sys.argv[-1]}")
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fix a regression introduced in 3.14.3 and 3.13.12 where the
:mod:`multiprocessing` ``forkserver`` start method would fail with
:exc:`BrokenPipeError` when the parent process had a very large
:data:`sys.argv`. The argv is now passed to the forkserver as separate
command-line arguments rather than being embedded in the ``-c`` command
string, avoiding the operating system's per-argument length limit.
Loading