[MG_Remote, MG_Test, scripts] (P7): bound the pre-auth cap and pause accepting when out of descriptors

This commit is contained in:
2026-09-23 13:45:00 -04:00
parent bc62b17b0c
commit efc899a1b0
6 changed files with 115 additions and 11 deletions
+10 -3
View File
@@ -89,6 +89,11 @@ namespace MobileGL::MG_Remote::Server {
// port probe; one that held its slot longer is a failure. A real client writes its first frame // port probe; one that held its slot longer is a failure. A real client writes its first frame
// the moment it connects, so it is never on the wrong side of this. // the moment it connects, so it is never on the wrong side of this.
inline constexpr std::uint32_t kPreAuthProbeGraceMs = 250; inline constexpr std::uint32_t kPreAuthProbeGraceMs = 250;
// MOBILEGL_IPC_PREAUTH_MAX never exceeds this. Every pending connection is a descriptor the
// supervisor holds, and an unbounded knob would let a connection flood walk it into
// RLIMIT_NOFILE (the supervisor also survives that now - it pauses accepting - but a knob
// should not be the way in).
inline constexpr std::uint32_t kPreAuthMaxPendingCeiling = 256;
// The refusal details a peer (and the log) sees. Named once so the supervisor and the unit test // The refusal details a peer (and the log) sees. Named once so the supervisor and the unit test
// cannot drift on the words; the TCP lane's scripts mirror them rather than parse them, so a // cannot drift on the words; the TCP lane's scripts mirror them rather than parse them, so a
@@ -108,8 +113,9 @@ namespace MobileGL::MG_Remote::Server {
// MOBILEGL_IPC_PREAUTH_MS: how long a new connection has to present a complete first // MOBILEGL_IPC_PREAUTH_MS: how long a new connection has to present a complete first
// frame. A real client writes its Hello (or DataBind) the moment it connects. // frame. A real client writes its Hello (or DataBind) the moment it connects.
std::uint32_t deadlineMs = 2000; std::uint32_t deadlineMs = 2000;
// MOBILEGL_IPC_PREAUTH_MAX: connections the supervisor holds unauthenticated at once; one // MOBILEGL_IPC_PREAUTH_MAX: connections the supervisor holds unauthenticated at once, at
// more is refused Busy at accept. // most kPreAuthMaxPendingCeiling. When they are all taken a newcomer displaces the busiest
// address's oldest one, or is refused Busy at accept (ChoosePendingToDisplace).
std::uint32_t maxPending = 8; std::uint32_t maxPending = 8;
// MOBILEGL_IPC_AUTH_BACKOFF_AFTER: failures from one address before it is refused at // MOBILEGL_IPC_AUTH_BACKOFF_AFTER: failures from one address before it is refused at
// accept. 0 turns the backoff off (the fuzz arm does, to send many malformed frames from // accept. 0 turns the backoff off (the fuzz arm does, to send many malformed frames from
@@ -124,7 +130,8 @@ namespace MobileGL::MG_Remote::Server {
static PreAuthKnobs FromEnvironment() { static PreAuthKnobs FromEnvironment() {
PreAuthKnobs knobs; PreAuthKnobs knobs;
knobs.deadlineMs = Read("MOBILEGL_IPC_PREAUTH_MS", knobs.deadlineMs, false); knobs.deadlineMs = Read("MOBILEGL_IPC_PREAUTH_MS", knobs.deadlineMs, false);
knobs.maxPending = Read("MOBILEGL_IPC_PREAUTH_MAX", knobs.maxPending, false); knobs.maxPending =
std::min(Read("MOBILEGL_IPC_PREAUTH_MAX", knobs.maxPending, false), kPreAuthMaxPendingCeiling);
knobs.backoffAfter = Read("MOBILEGL_IPC_AUTH_BACKOFF_AFTER", knobs.backoffAfter, true); knobs.backoffAfter = Read("MOBILEGL_IPC_AUTH_BACKOFF_AFTER", knobs.backoffAfter, true);
knobs.backoffBaseMs = Read("MOBILEGL_IPC_AUTH_BACKOFF_MS", knobs.backoffBaseMs, false); knobs.backoffBaseMs = Read("MOBILEGL_IPC_AUTH_BACKOFF_MS", knobs.backoffBaseMs, false);
return knobs; return knobs;
+25 -3
View File
@@ -714,16 +714,23 @@ public:
std::vector<pollfd> fds; std::vector<pollfd> fds;
for (;;) { for (;;) {
Reap(); Reap();
fds.clear();
fds.push_back({m_listener, POLLIN, 0});
for (const auto& peer : m_pending) fds.push_back({peer.fd, POLLIN, 0});
auto now = Clock::now(); auto now = Clock::now();
// While accepting is paused (out of descriptors, AcceptNew) the listener sits out the
// poll - it stays readable, and would spin the loop - and the pause's end is a wakeup.
const bool acceptPaused = now < m_acceptResumeAt;
fds.clear();
fds.push_back({acceptPaused ? -1 : m_listener, POLLIN, 0});
for (const auto& peer : m_pending) fds.push_back({peer.fd, POLLIN, 0});
// 250 ms is the old accept timeout, i.e. how often a finished child is reaped while // 250 ms is the old accept timeout, i.e. how often a finished child is reaped while
// nothing arrives; a pending deadline that falls sooner shortens it. // nothing arrives; a pending deadline that falls sooner shortens it.
std::int64_t timeout = 250; std::int64_t timeout = 250;
for (const auto& peer : m_pending) for (const auto& peer : m_pending)
timeout = std::clamp<std::int64_t>( timeout = std::clamp<std::int64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(peer.deadline - now).count(), 0, timeout); std::chrono::duration_cast<std::chrono::milliseconds>(peer.deadline - now).count(), 0, timeout);
if (acceptPaused)
timeout = std::clamp<std::int64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(m_acceptResumeAt - now).count() + 1, 0,
timeout);
const int polled = ::poll(fds.data(), static_cast<nfds_t>(fds.size()), static_cast<int>(timeout)); const int polled = ::poll(fds.data(), static_cast<nfds_t>(fds.size()), static_cast<int>(timeout));
if (polled < 0 && errno != EINTR) { if (polled < 0 && errno != EINTR) {
WireLogError("MG_Remote server: supervisor poll failed: %s", std::strerror(errno)); WireLogError("MG_Remote server: supervisor poll failed: %s", std::strerror(errno));
@@ -823,6 +830,18 @@ private:
int fd = -1; int fd = -1;
const auto accepted = SocketTransport::AcceptOne(m_listener, 0, &fd); const auto accepted = SocketTransport::AcceptOne(m_listener, 0, &fd);
if (accepted == MOBILEGL_ERR_TIMEOUT) return 0; if (accepted == MOBILEGL_ERR_TIMEOUT) return 0;
if (accepted == MOBILEGL_ERR_OUT_OF_MEMORY) {
// f2-auth fix round. Out of descriptors (or memory) is not a dead listener: the
// connection waits in the backlog, and pending connections give descriptors back
// as they are judged or reach their deadline. The first cut returned 73 here, so
// a connection flood against a low RLIMIT_NOFILE (or a large pending cap) took the
// supervisor down. Now accepting pauses for kAcceptPauseMs and the loop goes on.
m_acceptResumeAt = now + std::chrono::milliseconds(kAcceptPauseMs);
WireLogError("MG_Remote server: accept paused for %u ms: out of descriptors or memory with %zu "
"connections awaiting authentication",
kAcceptPauseMs, m_pending.size());
return 0;
}
if (accepted != MOBILEGL_OK) return 73; if (accepted != MOBILEGL_OK) return 73;
std::string address = PeerAddress(fd); std::string address = PeerAddress(fd);
if (const std::uint32_t retry = m_backoff.RetryAfterMs(address, now); retry != 0) { if (const std::uint32_t retry = m_backoff.RetryAfterMs(address, now); retry != 0) {
@@ -979,6 +998,9 @@ private:
int m_handoff = -1; int m_handoff = -1;
std::vector<PendingPeer> m_pending; std::vector<PendingPeer> m_pending;
std::vector<PendingPeer> m_ready; std::vector<PendingPeer> m_ready;
// Accepting resumes at this time after accept(2) ran out of descriptors or memory.
static constexpr std::uint32_t kAcceptPauseMs = 100;
Clock::time_point m_acceptResumeAt{};
}; };
} }
@@ -486,7 +486,13 @@ namespace MobileGL::MG_Remote::Transport {
if (fd < 0) { if (fd < 0) {
// The same two non-failures AcceptPair re-polls on. // The same two non-failures AcceptPair re-polls on.
if (errno == EINTR || errno == ECONNABORTED) continue; if (errno == EINTR || errno == ECONNABORTED) continue;
WireLogError("MG_Remote SocketTransport: accept failed: %s", std::strerror(errno)); const int error = errno;
WireLogError("MG_Remote SocketTransport: accept failed: %s", std::strerror(error));
// Out of descriptors or memory is the PROCESS's state, not the listener's: the
// connection waits in the backlog (f2-auth fix round; the TCP supervisor pauses
// and retries instead of exiting 73).
if (error == EMFILE || error == ENFILE || error == ENOBUFS || error == ENOMEM)
return MOBILEGL_ERR_OUT_OF_MEMORY;
return MOBILEGL_ERR_TRANSPORT_CLOSED; return MOBILEGL_ERR_TRANSPORT_CLOSED;
} }
::fcntl(fd, F_SETFD, FD_CLOEXEC); ::fcntl(fd, F_SETFD, FD_CLOEXEC);
@@ -123,6 +123,10 @@ namespace MobileGL::MG_Remote::Transport {
static MobileGLResult ConnectDataConnection(const std::string& path, std::uint32_t timeoutMs, static MobileGLResult ConnectDataConnection(const std::string& path, std::uint32_t timeoutMs,
MobileGLByteSpan firstFrame, int* outFd); MobileGLByteSpan firstFrame, int* outFd);
// Accepts ONE connection (TCP options applied, CLOEXEC). MOBILEGL_ERR_TIMEOUT when none. // Accepts ONE connection (TCP options applied, CLOEXEC). MOBILEGL_ERR_TIMEOUT when none.
// MOBILEGL_ERR_OUT_OF_MEMORY when accept(2) ran out of descriptors or memory (EMFILE,
// ENFILE, ENOBUFS, ENOMEM): the connection stays in the backlog and the listener is fine,
// so a caller that serves many peers can wait and try again (the TCP supervisor does);
// TRANSPORT_CLOSED for every other failure.
static MobileGLResult AcceptOne(int listenFd, std::uint32_t timeoutMs, int* outFd); static MobileGLResult AcceptOne(int listenFd, std::uint32_t timeoutMs, int* outFd);
// Reads EXACTLY one framed message from a raw socket - the header, then the payload, // Reads EXACTLY one framed message from a raw socket - the header, then the payload,
// and not one byte more. A DataBind is followed on the same connection by StreamLink's // and not one byte more. A DataBind is followed on the same connection by StreamLink's
+10
View File
@@ -303,4 +303,14 @@ TEST(PreAuthGateTest, TheKnobsReadTheirEnvironmentAndKeepTheirDefaultsOtherwise)
EXPECT_EQ(knobs.backoffAfter, 5u); EXPECT_EQ(knobs.backoffAfter, 5u);
EXPECT_EQ(knobs.backoffBaseMs, 1000u); EXPECT_EQ(knobs.backoffBaseMs, 1000u);
} }
// f2-auth fix round: the pending cap is bounded, whatever it is set to - every pending
// connection is a descriptor. Red once by dropping the clamp: 100000 comes back.
{
ScopedEnvironment b("MOBILEGL_IPC_PREAUTH_MAX", "100000");
EXPECT_EQ(Server::PreAuthKnobs::FromEnvironment().maxPending, Server::kPreAuthMaxPendingCeiling);
}
{
ScopedEnvironment b("MOBILEGL_IPC_PREAUTH_MAX", "256");
EXPECT_EQ(Server::PreAuthKnobs::FromEnvironment().maxPending, 256u);
}
} }
+59 -4
View File
@@ -22,7 +22,9 @@ refusal (`data_bind_queued_before_bound`). The f2-auth fix round added that the
shared between addresses (`pending_queue_is_shared`) and that an address holding slots with silent shared between addresses (`pending_queue_is_shared`) and that an address holding slots with silent
connections is backed off (`silent_slot_holder_backed_off`), that routing DataBinds does not stall connections is backed off (`silent_slot_holder_backed_off`), that routing DataBinds does not stall
the supervisor (`data_bind_routing_does_not_stall`), and that a child which refuses its Hello the supervisor (`data_bind_routing_does_not_stall`), and that a child which refuses its Hello
refuses the DataBinds queued to it (`data_bind_queued_before_refused_hello`, ID-P7-44). The five refuses the DataBinds queued to it (`data_bind_queued_before_refused_hello`, ID-P7-44), and that
running out of descriptors pauses accepting instead of ending the supervisor
(`accept_out_of_descriptors`). The five
malformed-frame shapes are fuzz arm 1's own entry, scripts/ci/ph_fuzz_control_frames.py. malformed-frame shapes are fuzz arm 1's own entry, scripts/ci/ph_fuzz_control_frames.py.
It needs flatc, which is deliberately absent from the default build graph (gen_protocol.py's It needs flatc, which is deliberately absent from the default build graph (gen_protocol.py's
@@ -38,6 +40,7 @@ import json
import os import os
from pathlib import Path from pathlib import Path
import re import re
import resource
import shutil import shutil
import signal import signal
import socket import socket
@@ -234,18 +237,25 @@ def refused_listen(server, log, endpoint, token):
@contextmanager @contextmanager
def supervisor(server, log, same_build=False, serve=True, extra_env=None): def supervisor(server, log, same_build=False, serve=True, extra_env=None, nofile=None):
"""A supervisor on its own free loopback port. `serve=False` is the single-session shape """A supervisor on its own free loopback port. `serve=False` is the single-session shape
(no fork; the process IS the session and exits with it), which `single_session_listener` (no fork; the process IS the session and exits with it), which `single_session_listener`
below needs and nothing else does. `extra_env` sets supervisor knobs (PH-7 (5)'s pre-auth below needs and nothing else does. `extra_env` sets supervisor knobs (PH-7 (5)'s pre-auth
deadline, pending cap and backoff) for the controls that need non-default ones.""" deadline, pending cap and backoff) for the controls that need non-default ones. `nofile`
lowers the supervisor's soft RLIMIT_NOFILE, for the descriptor-exhaustion control."""
port = free_loopback_port() port = free_loopback_port()
base = log.with_suffix('.library.log') base = log.with_suffix('.library.log')
env = supervisor_environment(base, kToken, same_build=same_build) env = supervisor_environment(base, kToken, same_build=same_build)
env.update(extra_env or {}) env.update(extra_env or {})
def limit():
if nofile is not None:
resource.setrlimit(resource.RLIMIT_NOFILE, (nofile, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))
with log.open('wb') as output: with log.open('wb') as output:
child = subprocess.Popen([server, f'tcp://127.0.0.1:{port}'] + (['--serve'] if serve else []), child = subprocess.Popen([server, f'tcp://127.0.0.1:{port}'] + (['--serve'] if serve else []),
env=env, stdout=output, stderr=output, start_new_session=True) env=env, stdout=output, stderr=output, start_new_session=True,
preexec_fn=limit if nofile is not None else None)
try: try:
# Log.cpp's RoleLogPath: the base name is a BASE, and the server role writes # Log.cpp's RoleLogPath: the base name is a BASE, and the server role writes
# `<stem>.server<ext>`. Yielded rather than recomputed at the call site, so the one # `<stem>.server<ext>`. Yielded rather than recomputed at the call site, so the one
@@ -972,6 +982,51 @@ def main():
l.strip() for l in named.splitlines() if reason in l)} l.strip() for l in named.splitlines() if reason in l)}
assert process.poll() is None, 'the pre-auth limits killed the supervisor' assert process.poll() is None, 'the pre-auth limits killed the supervisor'
# ---- f2-auth fix round: OUT OF DESCRIPTORS, THE SUPERVISOR PAUSES ACCEPTING, NOT EXITS ----
#
# accept(2) failing with EMFILE used to be "the listener failed": the supervisor logged
# its summary and exited 73, so a connection flood against a low RLIMIT_NOFILE (or a large
# MOBILEGL_IPC_PREAUTH_MAX - now clamped to 256) took the server down for everybody. Here
# the supervisor runs with a soft limit of 16 descriptors (five in use when idle) and a
# pending cap it cannot reach (64), and 48 silent connections are opened (the listen
# backlog is 8, so they arrive in waves - the first waves alone pass the limit): it must
# name the pause, stay up, and - once the flood is gone - still answer a Hello
# (fingerprint 0, so the answer is the child's Refuse{WireFingerprint}). The backoff is
# off: the flood's failures are not this control's subject. RED before the fix: the
# supervisor has exited.
exhaust = {'MOBILEGL_IPC_PREAUTH_MAX': '64', 'MOBILEGL_IPC_PREAUTH_MS': '3000',
'MOBILEGL_IPC_AUTH_BACKOFF_AFTER': '0'}
with supervisor(args.server, args.out / 'descriptors.log', extra_env=exhaust, nofile=16) as (
port, process, serverLog):
flood = []
try:
# The first connect waits for the supervisor to be listening (connect_control's
# retry); the rest are non-blocking: a blocking one stalls this loop for its whole
# timeout once the backlog is full, and the flood would never outrun the deadlines.
flood.append(connect_control(port))
for _ in range(47):
peer = socket.socket()
flood.append(peer)
peer.setblocking(False)
try:
peer.connect(('127.0.0.1', port))
except (BlockingIOError, ConnectionRefusedError):
pass # refused is the red shape: the supervisor is gone
paused = wait_for_log(serverLog, 'accept paused for', seconds=5)
alive = process.poll() is None
finally:
for peer in flood:
peer.close()
assert alive, f'the supervisor exited ({process.poll()}) when accept ran out of descriptors; see {serverLog}'
assert paused is not None, f'the supervisor never named its accept pause in {serverLog}'
time.sleep(1.0) # every flood connection has been read as closed (or is still in the backlog)
after_flood = exchange(port, schema, hello(schema, flatbuffers))
assert after_flood.get('code') == 2 and after_flood.get('expected', 0), (
f'after the descriptor flood a Hello was answered {after_flood}')
assert process.poll() is None, 'the supervisor died after the descriptor flood'
evidence['accept_out_of_descriptors'] = {'line': next(
l.strip() for l in paused.splitlines() if 'accept paused for' in l), 'afterFlood': after_flood}
# ---- ID-P7-44, f2-auth fix round: A CHILD THAT REFUSES ITS HELLO REFUSES WHAT WAS QUEUED -- # ---- ID-P7-44, f2-auth fix round: A CHILD THAT REFUSES ITS HELLO REFUSES WHAT WAS QUEUED --
# #
# The supervisor forwards DataBinds to a child from the moment it forks it. A child that # The supervisor forwards DataBinds to a child from the moment it forks it. A child that