mirror of
https://github.com/MobileGL-Dev/MobileGL
synced 2026-11-20 08:21:56 +09:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a078b6bcee | ||
|
|
6bccf98d7f | ||
|
|
3a6c5c7f04 | ||
|
|
b017ec1c1f |
+1
-1
@@ -594,7 +594,7 @@ namespace MobileGL::MG_Config {
|
||||
// caller was fire-and-forget. The window's own creates are fire-and-forget AND read, so a
|
||||
// flag keyed on wantReply would disable the very mechanism it exists to enable
|
||||
// (docs/Disaggregated/notes/p12/CREATE-WINDOW-MEASURED.md sections 7-11).
|
||||
Uint32 CreateWindow = 6;
|
||||
Uint32 CreateWindow = 10;
|
||||
// MOBILEGL_IPC_ADOPT_TIER: 2 = emulate (client keeps the shadow and pushes), which
|
||||
// is the only tier P5 implements and the reason persistent-map-push can be non-zero
|
||||
// at all (R-6). 0 and 1 parse and are Fatal at use with "P11"; they exist now so the
|
||||
|
||||
@@ -420,7 +420,7 @@ namespace MobileGL::MG_ConfigLoader {
|
||||
// declared answers, and the device sweep showed the wall clock still falling at the old
|
||||
// cap of 4 (see WireTables.cpp's kCreateWindowMax). WireTables.cpp's static_assert and
|
||||
// StreamLink's crossing log are the checks on this number.
|
||||
ipc.CreateWindow = QueryEnvUint32("MOBILEGL_IPC_CREATE_WINDOW", 6, 1, 8);
|
||||
ipc.CreateWindow = QueryEnvUint32("MOBILEGL_IPC_CREATE_WINDOW", 10, 1, 12);
|
||||
// The verify harness compares the pushed block against the applier per verb; a
|
||||
// batched queue lets the comparer read a supplied field mid-apply, which is a
|
||||
// torn read rather than a divergence. The batch is therefore off whenever the
|
||||
|
||||
@@ -215,6 +215,32 @@ void main() { oColor = texture(uTex, vUv); }
|
||||
<< " (" << image.ColorName(cx, cy) << ")" << std::endl;
|
||||
}
|
||||
|
||||
// A REAL BOUNDARY BEFORE ANY WHITE-BOX READ OF THE APPLIER (P12).
|
||||
//
|
||||
// The reads below (RecordIsReadable and its callers) read state that lives in the
|
||||
// APPLIER, and the records they assert about are the mask moves a sticky-bit change
|
||||
// produces - `resource_respecify` rows. P12 made that row FIRE-AND-FORGET for a texture
|
||||
// whose existence is already confirmed (PipeRoute.h's
|
||||
// MGPipeResourceRespecifyWantsItsReply: the answer only ever said "the applier holds a
|
||||
// record for this handle", which the client's own publication latch already answers), so
|
||||
// the client no longer waits, and a read taken immediately after the setter now races
|
||||
// the apply thread. MEASURED: DirectGLES.P4aFinalFixScenario's DSA case red with
|
||||
// `Serial` 7 vs 7 and the RENDER_TARGET bit clear, twice in a row, on the inproc arm -
|
||||
// and it is exactly the failure the CtWireScenario death gates had, from the same cause.
|
||||
//
|
||||
// A READBACK IS THE BOUNDARY THAT HOLDS ON EVERY ARM: ReadPixels is
|
||||
// kReplySlot|kWaitReply, the client blocks on its answer, and an answer is posted only
|
||||
// after its record has been APPLIED - so every record published before it, the mask move
|
||||
// included, is applied too. (The harness's applied-seq fence would be the other choice
|
||||
// and it is right for inproc only; it times out on the spawn/tcp arms, which this file
|
||||
// has no registrations on today but may grow.)
|
||||
void FenceTheApplierRead() {
|
||||
GLubyte fencePixel[4]{};
|
||||
glReadPixels(0, 0, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, fencePixel);
|
||||
ASSERT_EQ(FirstGLError(), GLenum(GL_NO_ERROR))
|
||||
<< "the applier-read fence left a GL error behind";
|
||||
}
|
||||
|
||||
// The white-box gate of the M-A case: true when the applier holds a record for the
|
||||
// texture in this process. Prints the decline.
|
||||
bool RecordIsReadable(unsigned glTextureName, const char* what, PipeTextureResourceRecordPeek* out) {
|
||||
@@ -563,6 +589,7 @@ void main() { oColor = texture(uTex, vUv); }
|
||||
(void)uploadsReadable;
|
||||
glBindImageTexture(0, texture, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGBA8);
|
||||
ASSERT_EQ(FirstGLError(), GLenum(GL_NO_ERROR));
|
||||
FenceTheApplierRead();
|
||||
if (readable) {
|
||||
ASSERT_TRUE(PeekPipeTextureResourceRecord(texture, &record));
|
||||
EXPECT_EQ(record.ImageBindableHint, 1u)
|
||||
@@ -637,6 +664,7 @@ void main() { oColor = texture(uTex, vUv); }
|
||||
glCreateFramebuffers(1, &namedFbo);
|
||||
glNamedFramebufferTexture(namedFbo, GL_COLOR_ATTACHMENT0, early, 0);
|
||||
ASSERT_EQ(FirstGLError(), GLenum(GL_NO_ERROR));
|
||||
FenceTheApplierRead();
|
||||
if (earlyReadable) {
|
||||
ASSERT_TRUE(PeekPipeTextureResourceRecord(early, &earlyRecord));
|
||||
EXPECT_NE(earlyRecord.BindMask & (1u << 7), 0u)
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include <MG_Remote/FatalFunnel.h>
|
||||
#include <MG_Remote/Handshake.h>
|
||||
#include <MG_Remote/Transport/LinkMetrics.h>
|
||||
#include <MG_Remote/Transport/StreamLink.h>
|
||||
|
||||
#include "../CapsCodec.h"
|
||||
#include "../Protocol/generated/protocol_generated.h"
|
||||
@@ -2260,6 +2261,16 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// overlap and at most one frame of added latency (ruling 4 / ID-92).
|
||||
const Uint64 awaited = serial - credit;
|
||||
++m_presentCreditWaits;
|
||||
// P65CLIENTPACE: WHERE THE CLIENT'S FRAME GOES, ONE LINE PER FRAME.
|
||||
//
|
||||
// The frame is a lockstep ping-pong and every stage measured so far is IDLE: the client
|
||||
// GL thread at 13% of the frame, the server's apply at 7%, and the server's io thread
|
||||
// 97% blocked in recv waiting for this side. So the missing time is on the CLIENT, and
|
||||
// this is the only per-frame wait the client is known to take - it blocks here until
|
||||
// the server acknowledges the swap of (serial - credit). With it, the io thread's send
|
||||
// side (is the socket full?) and the frame's own wall/CPU, the 80% has nowhere left to
|
||||
// hide: it is either this wait, the send, or the game.
|
||||
const auto paceStarted = std::chrono::steady_clock::now();
|
||||
const BarrierWaitScope waiting;
|
||||
Transport::SessionWait wait = Transport::SessionWait::TimedOut;
|
||||
for (;;) {
|
||||
@@ -2286,6 +2297,17 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// ShutDown is teardown and returns: the present did not happen, the same answer
|
||||
// the barrier gives on a dead doorbell.
|
||||
if (wait == Transport::SessionWait::Reached) DrainEventRing(*m_events);
|
||||
const double creditMs =
|
||||
static_cast<double>(std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - paceStarted).count()) / 1e6;
|
||||
const auto io = Transport::StreamLink::TakeSendStats();
|
||||
MGLOG_I("P65ClientPace present=%llu credit_wait_ms=%.1f | io: %llu B in %llu sendmsg "
|
||||
"send_ms=%.1f total_ms=%.1f",
|
||||
static_cast<unsigned long long>(serial), creditMs,
|
||||
static_cast<unsigned long long>(io.bytes),
|
||||
static_cast<unsigned long long>(io.calls),
|
||||
static_cast<double>(io.nsInRecv) / 1e6,
|
||||
static_cast<double>(io.nsTotal) / 1e6);
|
||||
}
|
||||
m_presentsSent = serial;
|
||||
return serial;
|
||||
|
||||
@@ -398,7 +398,7 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// (THE NARRATIVE ABOVE PREDATES RETENTION and ends at "the default is 1". It is kept as
|
||||
// the record of what was tried; what made the window work is the declared-answer store,
|
||||
// and the default is 2 - Config.h carries the device numbers.)
|
||||
constexpr Uint32 kCreateWindowMax = 8;
|
||||
constexpr Uint32 kCreateWindowMax = 12;
|
||||
// THE POOL ARGUMENT THAT USED TO PIN THIS AT 4 IS GONE, AND WHY IT IS GONE IS THE ONE
|
||||
// THING TO RE-CHECK BEFORE RAISING IT AGAIN. It read: the pool is kDefaultReplySlotCount
|
||||
// slots addressed seq % count, so an answer is readable only while its slot has not been
|
||||
@@ -451,6 +451,32 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// answers from existing at all - the second is what a no-reply bit keyed on wantReply
|
||||
// would produce, and it would leave every other reading green.
|
||||
Uint64 g_createWindowTaken = 0;
|
||||
// P12 MEASUREMENT: WHY A CREATE DOES NOT DEFER. The window's own arithmetic says a frame's
|
||||
// blocking takes should be creates/window (4,536/6 = 756 at the shipped default), and the
|
||||
// device measures 1,155 - a CONSTANT ~400 that does not move when the window deepens (at
|
||||
// window 8 the ideal is 567 and the measurement is 859). A constant is a fixed population
|
||||
// rather than a queueing effect, and the four exits below are the only ways into it, so
|
||||
// counting them by name is what says which one it is. It matters because the frame turned
|
||||
// out to be LATENCY-bound, not throughput-bound (the server applies all 103,097 records in
|
||||
// 1.95 s of a 9.64 s frame, and the client's 1,161 blocking takes at ~4.9 ms are 5.7 s of
|
||||
// it) - so the way to make it faster is fewer blocks, not a faster apply.
|
||||
Uint64 g_blockNoWindow = 0, g_blockSuspectsFull = 0, g_blockSuspectObject = 0,
|
||||
g_blockWindowStillFull = 0, g_blockTotal = 0;
|
||||
void NoteCreateBlocked(Uint64& reason) {
|
||||
++reason;
|
||||
++g_blockTotal;
|
||||
// Once per 256 blocks: a handful of lines per load frame, and the composition is
|
||||
// readable from any one of them.
|
||||
if (g_blockTotal % 256 == 0) {
|
||||
MGLOG_I("P12 create blocks: total=%llu no-window=%llu suspects-full=%llu "
|
||||
"suspect-object=%llu window-still-full=%llu",
|
||||
static_cast<unsigned long long>(g_blockTotal),
|
||||
static_cast<unsigned long long>(g_blockNoWindow),
|
||||
static_cast<unsigned long long>(g_blockSuspectsFull),
|
||||
static_cast<unsigned long long>(g_blockSuspectObject),
|
||||
static_cast<unsigned long long>(g_blockWindowStillFull));
|
||||
}
|
||||
}
|
||||
|
||||
// A HANDLE WHOSE CREATE CAME BACK DECLINED IS NEVER WINDOWED AGAIN. Without this the window
|
||||
// would UNDO the repair it exists to keep: every later use of the object sends another create,
|
||||
@@ -624,6 +650,13 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// all four take the blocking branch, which is the pre-window shape - and on the device
|
||||
// this is the branch the whole row takes, because the deferral above it cannot be
|
||||
// collected. See the section comment for the measurement and the arithmetic.
|
||||
if (window <= 1) {
|
||||
NoteCreateBlocked(g_blockNoWindow);
|
||||
} else if (g_createSuspects.size() >= kCreateSuspectMax) {
|
||||
NoteCreateBlocked(g_blockSuspectsFull);
|
||||
} else if (CreateIsSuspect(payload->Resource)) {
|
||||
NoteCreateBlocked(g_blockSuspectObject);
|
||||
}
|
||||
if (window <= 1 || g_createSuspects.size() >= kCreateSuspectMax ||
|
||||
CreateIsSuspect(payload->Resource)) {
|
||||
// THE WINDOW RETIRES HERE, AND IT MAY BE HOLDING ANSWERS (P12). This branch is
|
||||
@@ -650,6 +683,7 @@ namespace MobileGL::MG_Remote::Client {
|
||||
// against the effective window, which is what the array below is sized for, so
|
||||
// this line IS the overwrite guard.
|
||||
if (g_createWindowCount >= window) {
|
||||
NoteCreateBlocked(g_blockWindowStillFull);
|
||||
EmitCreateBlocking(session, payload, reply);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
#include "ServerLoop.h"
|
||||
#include <MG_Remote/FatalFunnel.h>
|
||||
#include <MG_Remote/Transport/StreamLink.h>
|
||||
#include <MG_Backend/MGPipe/PipeInputs.h>
|
||||
|
||||
#include <Config.h>
|
||||
@@ -739,6 +740,30 @@ namespace MobileGL::MG_Remote::Server {
|
||||
ServerSession& session = *m_session;
|
||||
Transport::SessionConsumer& consumer = session.Consumer();
|
||||
PipeApplier& applier = session.Applier();
|
||||
// ---- P65ServerFrame: WHERE THE SERVER'S TIME GOES (P12 measurement) -----------------
|
||||
//
|
||||
// THE CLIENT SIDE IS MEASURED OUT AND THE SERVER'S IS NOT MEASURED AT ALL. Every reading
|
||||
// this harness produces is the CLIENT's - wall_ns, wait_replies, client_thread_cpu_ns - and
|
||||
// they now add up to less than the frame: at window 6 the client burns 2.5 s of CPU and
|
||||
// 1,162 blocking waits, and removing 292 of those waits (window 8) moves the wall clock by
|
||||
// nothing. The same 101 MB over USB instead of WiFi moves it by nothing either. What is
|
||||
// left is the one process nobody has timed: the server applying 103,097 records.
|
||||
//
|
||||
// THE SPLIT IS THE POINT. apply_ms is the sum of the time spent INSIDE
|
||||
// PipeApplier::ApplyOne - the GL work, the decode, the reply posts - and wall_ms is Present
|
||||
// to Present on the apply thread. Their difference is everything that is NOT applying:
|
||||
// waiting on the ring for the next record, retiring, and the socket. So one line answers
|
||||
// "is the phone CPU-bound, or is it waiting for the client?" without a profiler, which is
|
||||
// the question the next optimisation has to be chosen from.
|
||||
//
|
||||
// FUNCTION-LOCAL STATIC BECAUSE THE FRAME DOES NOT FIT IN ONE DrainRing CALL: records
|
||||
// trickle in, so a frame spans many drains, and state that reset per drain would measure a
|
||||
// batch. There is exactly one apply thread (ServerLoopInstance's), which is what makes a
|
||||
// static safe here; the same reasoning the drain tally above uses.
|
||||
static auto sFrameWallStart = std::chrono::steady_clock::now();
|
||||
static std::uint64_t sFrameApplyNs = 0;
|
||||
static std::uint64_t sFrameRecords = 0;
|
||||
static std::uint64_t sFrameIndex = 0;
|
||||
Uint64 applied = 0;
|
||||
for (;;) {
|
||||
if (SessionLatched()) break;
|
||||
@@ -749,7 +774,50 @@ namespace MobileGL::MG_Remote::Server {
|
||||
bool corrupt = false;
|
||||
const bool popped = consumer.ApplyOne(
|
||||
[this, &applier](const Transport::RingRecordView& record) {
|
||||
const auto applyStarted = std::chrono::steady_clock::now();
|
||||
applier.ApplyOne(record);
|
||||
// P65ServerFrame: timed around the ONE call that is the server's work, and
|
||||
// summed across the drains a frame spans. See the block above DrainRing's loop.
|
||||
sFrameApplyNs += static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - applyStarted)
|
||||
.count());
|
||||
++sFrameRecords;
|
||||
if (record.kind == static_cast<std::uint16_t>(MG_Pipe::MGPWireOp::Present)) {
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
const auto wallNs = static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(now - sFrameWallStart)
|
||||
.count());
|
||||
const double applyMs = static_cast<double>(sFrameApplyNs) / 1e6;
|
||||
const double wallMs = static_cast<double>(wallNs) / 1e6;
|
||||
// AND THE STAGE NOTHING ELSE COVERS: the io thread's socket reads, which
|
||||
// its own MGLOG_ lines cannot report (the phone's forwarded log carries
|
||||
// mgl-srv-apply and mgl-display-ser only). read_ms is wall time inside
|
||||
// recv - blocking included - so a frame that is 90% read_ms is a frame the
|
||||
// reader spent WAITING for bytes, and one that is mostly recv calls with
|
||||
// little read_ms is a reader that is copying.
|
||||
const auto read = Transport::StreamLink::TakeReadStats();
|
||||
MGLOG_I("P65ServerFrame frame=%llu records=%llu apply_ms=%.1f wall_ms=%.1f "
|
||||
"outside_ms=%.1f | read: %llu B in %llu recv(%llu reads) "
|
||||
"recv_ms=%.1f total_ms=%.1f (%.0f%% blocked, %.1f MB/s)",
|
||||
static_cast<unsigned long long>(++sFrameIndex),
|
||||
static_cast<unsigned long long>(sFrameRecords), applyMs, wallMs,
|
||||
wallMs - applyMs,
|
||||
static_cast<unsigned long long>(read.bytes),
|
||||
static_cast<unsigned long long>(read.calls),
|
||||
static_cast<unsigned long long>(read.reads),
|
||||
static_cast<double>(read.nsInRecv) / 1e6,
|
||||
static_cast<double>(read.nsTotal) / 1e6,
|
||||
read.nsTotal ? 100.0 * static_cast<double>(read.nsInRecv) /
|
||||
static_cast<double>(read.nsTotal)
|
||||
: 0.0,
|
||||
read.nsTotal ? static_cast<double>(read.bytes) /
|
||||
(static_cast<double>(read.nsTotal) / 1e9) / 1e6
|
||||
: 0.0);
|
||||
sFrameWallStart = now;
|
||||
sFrameApplyNs = 0;
|
||||
sFrameRecords = 0;
|
||||
}
|
||||
// THE TALLY MOVES HERE, INSIDE THE CALLBACK, AND NOT AFTER THE BATCH. s1's
|
||||
// SessionConsumer::ApplyOne publishes appliedSeq the instant this callback
|
||||
// returns, and publishing appliedSeq is what releases the client from the
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
namespace {
|
||||
struct Window {
|
||||
std::uint64_t waitReplies = 0, samples = 0, replyNs = 0, stageBytes = 0;
|
||||
std::uint64_t transportWaitNs = 0;
|
||||
std::array<std::uint64_t, 32> histogram{};
|
||||
// One slot per wire op, plus a last slot for anything past the table. See
|
||||
// LinkMetricsMaxOps: the point is that the per-op numbers still SUM to waitReplies.
|
||||
@@ -92,14 +93,16 @@ namespace {
|
||||
}
|
||||
MGLOG_I("P65LinkMetrics kind=%s frame=%llu wait_replies=%llu rtt_samples=%llu "
|
||||
"rtt_mean_us=%.3f rtt_p50_upper_us=%llu rtt_p99_upper_us=%llu "
|
||||
"stage_bytes=%llu wall_ns=%llu client_thread_cpu_ns=%llu rtt_hist_us_pow2=%s",
|
||||
"stage_bytes=%llu wall_ns=%llu client_thread_cpu_ns=%llu "
|
||||
"transport_wait_ns=%llu rtt_hist_us_pow2=%s",
|
||||
kind, static_cast<unsigned long long>(metrics.frame),
|
||||
static_cast<unsigned long long>(w.waitReplies), static_cast<unsigned long long>(w.samples),
|
||||
w.samples == 0 ? 0.0 : double(w.replyNs) / double(w.samples) / 1000.0,
|
||||
static_cast<unsigned long long>(QuantileUpperUs(w, 50)),
|
||||
static_cast<unsigned long long>(QuantileUpperUs(w, 99)),
|
||||
static_cast<unsigned long long>(w.stageBytes), static_cast<unsigned long long>(wallNs),
|
||||
static_cast<unsigned long long>(cpuNs), histogram);
|
||||
static_cast<unsigned long long>(cpuNs),
|
||||
static_cast<unsigned long long>(w.transportWaitNs), histogram);
|
||||
// AND THE PER-OP SPLIT WITH EVERY FRAME. A whole-session total cannot name the rows behind
|
||||
// ONE frame, and one frame is the whole shape of this problem: the device run that
|
||||
// motivated this printed 43,232 of its 43,629 waits in a single atlas frame, and the exit
|
||||
@@ -167,6 +170,15 @@ void LinkMetricsStageBytes(std::uint64_t bytes) {
|
||||
if (!metrics.active) return;
|
||||
metrics.current.stageBytes += bytes; metrics.total.stageBytes += bytes;
|
||||
}
|
||||
std::uint64_t LinkMetricsBeginTransportWait() {
|
||||
return metrics.active ? ClockNs() : 0;
|
||||
}
|
||||
void LinkMetricsEndTransportWait(std::uint64_t startedNs) {
|
||||
if (!metrics.active || startedNs == 0) return;
|
||||
const auto elapsed = ClockNs() - startedNs;
|
||||
metrics.current.transportWaitNs += elapsed;
|
||||
metrics.total.transportWaitNs += elapsed;
|
||||
}
|
||||
void LinkMetricsPresent() {
|
||||
if (!metrics.active) return;
|
||||
++metrics.frame;
|
||||
|
||||
@@ -17,6 +17,11 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
std::uint64_t LinkMetricsBeginReply(bool wantsReply, std::uint32_t op);
|
||||
void LinkMetricsReplyApplied(std::uint64_t startedNs);
|
||||
void LinkMetricsStageBytes(std::uint64_t bytes);
|
||||
// Wall time spent in the producer's transport park (flush, spin and sleep).
|
||||
// The server can run concurrently, so this is an independently measured
|
||||
// duration, not a residual obtained by subtracting role CPU clocks.
|
||||
std::uint64_t LinkMetricsBeginTransportWait();
|
||||
void LinkMetricsEndTransportWait(std::uint64_t startedNs);
|
||||
|
||||
// THE SAME NUMBER THE PER-FRAME LINE PRINTS AS `wait_replies` (P65LinkMetrics), read back
|
||||
// in-process. P12 item B needs it as a GATE rather than as a log line: "the texture half no
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "Ring.h"
|
||||
|
||||
#include "SessionRings.h"
|
||||
#include "LinkMetrics.h"
|
||||
#include "WireLog.h"
|
||||
|
||||
#include <MG_Util/Debug/Log.h>
|
||||
@@ -472,21 +473,27 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
if (!Valid() || m_selfBell == nullptr) {
|
||||
return SessionWait::TimedOut;
|
||||
}
|
||||
const auto transportStarted = LinkMetricsBeginTransportWait();
|
||||
// COUNTED BEFORE THE WAIT AND ONLY WHEN ONE REALLY HAPPENS - after the two refusals
|
||||
// above, which answer TimedOut without ever reaching the bell. See Waits()/Parks() in
|
||||
// the header for why the pair is the reading and either number alone is not.
|
||||
if (m_link && m_link->Flush() != MOBILEGL_OK) return SessionWait::ShutDown;
|
||||
if (m_link && m_link->Flush() != MOBILEGL_OK) {
|
||||
LinkMetricsEndTransportWait(transportStarted);
|
||||
return SessionWait::ShutDown;
|
||||
}
|
||||
m_waits.fetch_add(1, std::memory_order_relaxed);
|
||||
// &m_parks is the OTHER half of the pair, and it is passed rather than read off the
|
||||
// bell: the bell is this endpoint's and the encoder waits on it too, so its own
|
||||
// ParkEntries() is not a subset of the waits counted one line above. See Parks() in
|
||||
// SessionRings.h.
|
||||
if (m_selfBell->Wait(m_control->producerParked, ready, m_spinUs, timeoutMs, &m_parks)) {
|
||||
LinkMetricsEndTransportWait(transportStarted);
|
||||
return SessionWait::Reached;
|
||||
}
|
||||
// Wait == false && Dead() is "the session was shut down", and it is the
|
||||
// only thing that returns from a kWaitForever park. Anything else is the
|
||||
// deadline.
|
||||
LinkMetricsEndTransportWait(transportStarted);
|
||||
return m_selfBell->Dead() ? SessionWait::ShutDown : SessionWait::TimedOut;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,16 @@
|
||||
#endif
|
||||
|
||||
namespace MobileGL::MG_Remote::Transport {
|
||||
// P65Read's counters: written by the server's io thread, drained by its apply thread (see
|
||||
// StreamLink::TakeReadStats).
|
||||
static std::atomic<std::uint64_t> gReadBytes{0}, gReadCalls{0}, gReadReads{0};
|
||||
static std::atomic<std::uint64_t> gReadNsInRecv{0}, gReadNsTotal{0};
|
||||
// P65CLIENTPACE's send half: the io thread's side of the same question. A sendmsg that blocks
|
||||
// is the socket full, i.e. the WRITE side's back-pressure - and on a client whose frame is 80%
|
||||
// waiting, "is it waiting to hand bytes over, or waiting for the game to make them" is the
|
||||
// whole question. Time spent inside sendmsg, not calls made.
|
||||
static std::atomic<std::uint64_t> gSendBytes{0}, gSendCalls{0}, gSendNsInSend{0};
|
||||
static std::atomic<std::uint64_t> gSendNsTotal{0};
|
||||
namespace {
|
||||
constexpr std::uint64_t kDataEnvelopeBytes = 20;
|
||||
constexpr std::uint64_t kMaxDataPayload = kMaxFramePayloadSize - kDataEnvelopeBytes;
|
||||
@@ -187,9 +197,32 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
}
|
||||
bool Read(void* data, std::size_t size) {
|
||||
#if !defined(_WIN32)
|
||||
// P65Read: THE SERVER'S SOCKET-READ PATH, WHICH NOTHING MEASURED. The per-frame
|
||||
// clocks say the frame's time is NOT in the client's GL thread (13%) and NOT in
|
||||
// PipeApplier::ApplyOne (11%) - 76% of a 105-185 ms frame is outside both, and the
|
||||
// only stage between them is this loop. Two different worlds look identical from
|
||||
// outside it and need opposite fixes:
|
||||
// * few BIG recvs with little time blocked -> the reader is copying, i.e. one
|
||||
// thread's throughput is the ceiling;
|
||||
// * many SMALL recvs, most of the wall spent blocked -> the reader is STARVED and
|
||||
// the client (or the batching on its side) sets the pace.
|
||||
// Counted here rather than sampled from /proc because the question is per-call, not
|
||||
// per-thread. Printed every kReadMarkBytes so a frame's worth of traffic yields
|
||||
// several lines and no line can be lost to a quiet period.
|
||||
constexpr std::uint64_t kReadMarkBytes = 8ull * 1024ull * 1024ull;
|
||||
(void)kReadMarkBytes;
|
||||
const auto readStarted = std::chrono::steady_clock::now();
|
||||
std::uint64_t calls = 0;
|
||||
auto* p = static_cast<std::uint8_t*>(data);
|
||||
const auto wanted = size;
|
||||
while (size) {
|
||||
const auto recvStarted = std::chrono::steady_clock::now();
|
||||
const auto n = ::recv(fd, p, size, 0);
|
||||
gReadNsInRecv.fetch_add(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - recvStarted).count()),
|
||||
std::memory_order_relaxed);
|
||||
++calls;
|
||||
if (n < 0 && errno == EINTR) continue;
|
||||
if (n <= 0) {
|
||||
Die(!stopping.load(std::memory_order_acquire));
|
||||
@@ -198,6 +231,13 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
p += n;
|
||||
size -= static_cast<std::size_t>(n);
|
||||
}
|
||||
gReadCalls.fetch_add(calls, std::memory_order_relaxed);
|
||||
gReadReads.fetch_add(1, std::memory_order_relaxed);
|
||||
gReadBytes.fetch_add(wanted, std::memory_order_relaxed);
|
||||
gReadNsTotal.fetch_add(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - readStarted).count()),
|
||||
std::memory_order_relaxed);
|
||||
return true;
|
||||
#else
|
||||
(void)data;
|
||||
@@ -221,8 +261,16 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
msghdr message{};
|
||||
message.msg_iov = vectors;
|
||||
message.msg_iovlen = size ? 2 : 1;
|
||||
const auto sendStarted = std::chrono::steady_clock::now();
|
||||
std::uint64_t sendCalls = 0;
|
||||
while (message.msg_iovlen) {
|
||||
const auto syscallStarted = std::chrono::steady_clock::now();
|
||||
const auto sent = ::sendmsg(fd, &message, MSG_NOSIGNAL);
|
||||
gSendNsInSend.fetch_add(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - syscallStarted).count()),
|
||||
std::memory_order_relaxed);
|
||||
++sendCalls;
|
||||
if (sent < 0 && errno == EINTR) continue;
|
||||
if (sent <= 0) {
|
||||
Die(!stopping.load(std::memory_order_acquire));
|
||||
@@ -239,6 +287,12 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
message.msg_iov->iov_len -= consumed;
|
||||
}
|
||||
}
|
||||
gSendCalls.fetch_add(sendCalls, std::memory_order_relaxed);
|
||||
gSendBytes.fetch_add(size, std::memory_order_relaxed);
|
||||
gSendNsTotal.fetch_add(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - sendStarted).count()),
|
||||
std::memory_order_relaxed);
|
||||
return true;
|
||||
#else
|
||||
(void)payload;
|
||||
@@ -425,6 +479,25 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
}
|
||||
};
|
||||
|
||||
StreamLink::ReadStats StreamLink::TakeSendStats() {
|
||||
ReadStats out;
|
||||
out.bytes = gSendBytes.exchange(0, std::memory_order_relaxed);
|
||||
out.calls = gSendCalls.exchange(0, std::memory_order_relaxed);
|
||||
out.nsInRecv = gSendNsInSend.exchange(0, std::memory_order_relaxed);
|
||||
out.nsTotal = gSendNsTotal.exchange(0, std::memory_order_relaxed);
|
||||
return out;
|
||||
}
|
||||
|
||||
StreamLink::ReadStats StreamLink::TakeReadStats() {
|
||||
ReadStats out;
|
||||
out.bytes = gReadBytes.exchange(0, std::memory_order_relaxed);
|
||||
out.calls = gReadCalls.exchange(0, std::memory_order_relaxed);
|
||||
out.reads = gReadReads.exchange(0, std::memory_order_relaxed);
|
||||
out.nsInRecv = gReadNsInRecv.exchange(0, std::memory_order_relaxed);
|
||||
out.nsTotal = gReadNsTotal.exchange(0, std::memory_order_relaxed);
|
||||
return out;
|
||||
}
|
||||
|
||||
StreamLink::StreamLink() : m_impl(new Impl) {}
|
||||
StreamLink::~StreamLink() {
|
||||
Detach();
|
||||
|
||||
@@ -11,6 +11,23 @@ namespace MobileGL::MG_Remote::Transport {
|
||||
public:
|
||||
StreamLink();
|
||||
~StreamLink() override;
|
||||
// P65READ: THE SERVER'S SOCKET-READ PATH, ACCOUNTED WHERE IT CAN BE SEEN. The counters are
|
||||
// published from the io thread's recv loop and read by the APPLY thread, because the io
|
||||
// thread's own MGLOG_ lines never reach the phone's forwarded log (only mgl-srv-apply and
|
||||
// mgl-display-ser do - measured, not assumed). Taking them RESETS them, so one call is one
|
||||
// frame's worth of the stage nothing else measures: the frame's time is not the client's GL
|
||||
// thread (13%) and not PipeApplier::ApplyOne (11%), and this is what sits between them.
|
||||
struct ReadStats {
|
||||
std::uint64_t bytes = 0; // payload bytes taken off the socket
|
||||
std::uint64_t calls = 0; // recv() calls it took
|
||||
std::uint64_t reads = 0; // Link::Read() calls (one per header, envelope or chunk)
|
||||
std::uint64_t nsInRecv = 0; // wall time inside recv - blocking included
|
||||
std::uint64_t nsTotal = 0; // wall time inside Read
|
||||
};
|
||||
static ReadStats TakeReadStats();
|
||||
// The client's half: bytes handed to the io thread's sendmsg, the calls it took, and the
|
||||
// wall time inside sendmsg (blocking included). nsInRecv carries the send time.
|
||||
static ReadStats TakeSendStats();
|
||||
MobileGLResult Attach(int dataFd, SessionSegments& mirrors, TransportRoleTag role);
|
||||
MobileGLResult AttachOwned(int dataFd, const struct SessionSegmentSizes& sizes, TransportRoleTag role);
|
||||
// PH-7 (4), ID-P7-3. The SERVER's half of a nonce-bound data connection, in two steps.
|
||||
|
||||
@@ -563,6 +563,25 @@ by name (`NoServerDisplay`) and that service goes on serving.
|
||||
|
||||
## TCP credit and Stage measurements
|
||||
|
||||
For a paired frame-time breakdown, enable `MOBILEGL_PIPE_STATS=1` in both roles
|
||||
and keep their logs. The following reducer matches client and server present
|
||||
serials, drops the first (unpaired) server interval and one warmup frame, and
|
||||
reports avg/p50/p99 in milliseconds plus percentages of client present time:
|
||||
|
||||
```bash
|
||||
python3 tools/trace_replay/frame_stats.py \
|
||||
--client-log mobilegl.client.log --server-log mobilegl.server.log \
|
||||
--warmup-frames 1 --output frame-stats.json
|
||||
```
|
||||
|
||||
`transport_wait_ms` measures wall time in the client producer's transport park
|
||||
(flush, spin and sleep). It includes waiting for replies, present credit and
|
||||
ring space; it is not socket latency. Client and server CPU can overlap with
|
||||
each other and with a transport wait, so each percentage uses the client
|
||||
present interval as its denominator and the three percentages need not sum to
|
||||
100%. A missing frame is rejected. Percentiles use nearest rank over per-frame
|
||||
samples.
|
||||
|
||||
`benchmark_tcp_credits.py` runs the complete OpenRA and rd12 traces at credits
|
||||
1, 2, and 3, then selects matching client/server tail frame IDs (100 and 200
|
||||
frames respectively). Extract the unchanged trace files as
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pair client/server present intervals and summarize their frame time.
|
||||
|
||||
MOBILEGL_PIPE_STATS=1 enables the per-present clocks. Frame IDs are matched
|
||||
exactly; the first server interval is invalid because it has no preceding
|
||||
present. Transport wait measures wall time inside the client's transport park
|
||||
(flush, spin and sleep). It can overlap with server CPU and is not network latency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
FIELDS = re.compile(r"(\w+)=([^\s]+)")
|
||||
CLIENT_MARKER = "P65LinkMetrics "
|
||||
SERVER_MARKER = "P65ServerMetrics "
|
||||
|
||||
|
||||
def read_frames(path: Path, marker: str, *, server: bool) -> dict[int, dict]:
|
||||
rows = {}
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if marker not in line:
|
||||
continue
|
||||
values = dict(FIELDS.findall(line.split(marker, 1)[1]))
|
||||
if server and values.get("valid") != "1":
|
||||
continue
|
||||
if not server and values.get("kind") != "frame":
|
||||
continue
|
||||
try:
|
||||
number = int(values["frame"])
|
||||
wall = int(values["wall_ns"])
|
||||
cpu = int(values["apply_thread_cpu_ns" if server else "client_thread_cpu_ns"])
|
||||
transport = 0 if server else int(values["transport_wait_ns"])
|
||||
except (KeyError, ValueError) as error:
|
||||
raise ValueError(f"{path}: malformed {marker.strip()} row") from error
|
||||
if number in rows:
|
||||
raise ValueError(f"{path}: duplicate frame {number}")
|
||||
# The wall and thread-CPU clocks are sampled separately. Their nanosecond
|
||||
# readings can differ slightly at a boundary even for a fully busy thread.
|
||||
tolerance = max(100_000, wall // 100)
|
||||
if (number <= 0 or wall <= 0 or cpu <= 0 or cpu > wall + tolerance or
|
||||
transport < 0 or transport > wall + tolerance):
|
||||
raise ValueError(f"{path}: invalid clocks in frame {number}")
|
||||
rows[number] = {"wall_ns": wall, "cpu_ns": cpu, "transport_ns": transport}
|
||||
if not rows:
|
||||
raise ValueError(f"{path}: no usable {marker.strip()} frames")
|
||||
return rows
|
||||
|
||||
|
||||
def quantile_ns(samples: list[int], percentile: int) -> int:
|
||||
ordered = sorted(samples)
|
||||
return ordered[max(0, math.ceil(len(ordered) * percentile / 100) - 1)]
|
||||
|
||||
|
||||
def distribution(samples: list[int]) -> dict[str, float]:
|
||||
return {"avg": sum(samples) / len(samples) / 1e6,
|
||||
"p50": quantile_ns(samples, 50) / 1e6,
|
||||
"p99": quantile_ns(samples, 99) / 1e6}
|
||||
|
||||
|
||||
def summarize(client_path: Path, server_path: Path, *, warmup: int = 0,
|
||||
tail: int | None = None) -> dict:
|
||||
client = read_frames(client_path, CLIENT_MARKER, server=False)
|
||||
server = read_frames(server_path, SERVER_MARKER, server=True)
|
||||
selected = sorted(number for number in client if number > warmup)
|
||||
if tail is not None:
|
||||
selected = selected[-tail:]
|
||||
if not selected:
|
||||
raise ValueError("no frames remain after warmup/tail selection")
|
||||
if selected != list(range(selected[0], selected[-1] + 1)):
|
||||
raise ValueError("selected client frame IDs are not contiguous")
|
||||
missing = [number for number in selected if number not in server]
|
||||
if missing:
|
||||
raise ValueError(f"server is missing matched frames: {missing[:10]}")
|
||||
|
||||
columns = {name: [] for name in ("client_present_ms", "server_present_ms",
|
||||
"client_cpu_ms", "transport_wait_ms", "server_cpu_ms")}
|
||||
for number in selected:
|
||||
c, s = client[number], server[number]
|
||||
for key, value in (("client_present_ms", c["wall_ns"]),
|
||||
("server_present_ms", s["wall_ns"]),
|
||||
("client_cpu_ms", c["cpu_ns"]),
|
||||
("transport_wait_ms", c["transport_ns"]),
|
||||
("server_cpu_ms", s["cpu_ns"])):
|
||||
columns[key].append(value)
|
||||
|
||||
duration = sum(columns["client_present_ms"])
|
||||
shares = {name: 100 * sum(columns[name]) / duration
|
||||
for name in ("client_cpu_ms", "transport_wait_ms", "server_cpu_ms")}
|
||||
return {"frames": len(selected), "first_frame": selected[0],
|
||||
"last_frame": selected[-1], "warmup_frames": warmup,
|
||||
"definition": "transport_wait is client wall time in SessionProducer::Park "
|
||||
"(flush, spin and sleep); role CPU and transport wait can overlap",
|
||||
"percentile": "nearest rank", "unit": "ms",
|
||||
"stats": {key: distribution(values) for key, values in columns.items()},
|
||||
"share_percent_of_client_present": shares}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--client-log", type=Path, required=True)
|
||||
parser.add_argument("--server-log", type=Path, required=True)
|
||||
parser.add_argument("--warmup-frames", type=int, default=1)
|
||||
parser.add_argument("--tail-frames", type=int)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.warmup_frames < 0 or args.tail_frames is not None and args.tail_frames <= 0:
|
||||
parser.error("warmup must be nonnegative and tail must be positive")
|
||||
try:
|
||||
result = summarize(args.client_log, args.server_log, warmup=args.warmup_frames,
|
||||
tail=args.tail_frames)
|
||||
except (OSError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.write_text(rendered, encoding="utf-8")
|
||||
else:
|
||||
print(rendered, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small fixture checks for frame ID pairing and additive frame attribution."""
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from frame_stats import summarize
|
||||
|
||||
|
||||
class FrameStatsTest(unittest.TestCase):
|
||||
def logs(self, root, *, missing_server=False, overlapping=False):
|
||||
client = root / "client.log"
|
||||
server = root / "server.log"
|
||||
client.write_text("\n".join(
|
||||
f"P65LinkMetrics kind=frame frame={n} wall_ns={wall} "
|
||||
f"client_thread_cpu_ns={cpu} transport_wait_ns={wait}"
|
||||
for n, wall, cpu, wait in ((1, 10000000, 2000000, 1000000),
|
||||
(2, 20000000, 5000000, 4000000),
|
||||
(3, 40000000, 10000000, 7000000))) + "\n")
|
||||
server.write_text("\n".join(
|
||||
f"P65ServerMetrics frame={n} valid=1 wall_ns={wall} "
|
||||
f"apply_thread_cpu_ns={cpu}"
|
||||
for n, wall, cpu in ((2, 18000000, 17000000 if overlapping else 3000000),
|
||||
(3, 35000000, 31000000 if overlapping else 6000000))
|
||||
if not missing_server or n != 3) + "\n")
|
||||
return client, server
|
||||
|
||||
def test_paired_percentiles_and_shares(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = summarize(*self.logs(Path(directory)), warmup=1)
|
||||
self.assertEqual((result["first_frame"], result["last_frame"], result["frames"]),
|
||||
(2, 3, 2))
|
||||
self.assertEqual(result["stats"]["transport_wait_ms"],
|
||||
{"avg": 5.5, "p50": 4.0, "p99": 7.0})
|
||||
self.assertEqual(result["share_percent_of_client_present"]["transport_wait_ms"],
|
||||
100 * 11000000 / 60000000)
|
||||
|
||||
def test_missing_server_frame_is_error(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
with self.assertRaisesRegex(ValueError, "missing matched frames"):
|
||||
summarize(*self.logs(Path(directory), missing_server=True), warmup=1)
|
||||
|
||||
def test_parallel_cpu_is_allowed(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = summarize(*self.logs(Path(directory), overlapping=True), warmup=1)
|
||||
self.assertGreater(sum(result["share_percent_of_client_present"].values()), 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user