GCC Code Coverage Report


Directory: ./
File: mediaoutput/AES67Manager.h
Date: 2026-10-29 08:11:58
Exec Total Coverage
Lines: 12 16 75.0%
Functions: 6 6 100.0%
Branches: 11 18 61.1%

Line Branch Exec Source
1 #pragma once
2 /*
3 * This file is part of the Falcon Player (FPP) and is Copyright (C)
4 * 2013-2025 by the Falcon Player Developers.
5 *
6 * The Falcon Player (FPP) is free software, and is covered under
7 * multiple Open Source licenses. Please see the included 'LICENSES'
8 * file for descriptions of what files are covered by each license.
9 *
10 * This source file is covered under the LGPL v2.1 as described in the
11 * included LICENSE.LGPL file.
12 */
13
14 // AES67Manager — GStreamer-based AES67 audio-over-IP send/receive
15 //
16 // Replaces the previous PipeWire RTP module approach with GStreamer pipelines
17 // and uses ptp4l (linuxptp) for IEEE 1588 PTP clock synchronization on the
18 // network, achieving true AES67 compliance.
19 //
20 // Features:
21 // - Send: pipewiresrc → audioconvert → rtpL24pay → udpsink
22 // - Receive: udpsrc → rtpjitterbuffer → rtpL24depay → audioconvert → pipewiresink
23 // - ptp4l for IEEE 1588 PTP on the network (grandmaster or follower via BMCA)
24 // - Built-in SAP announcer (replaces external fpp_aes67_sap Python daemon)
25 // - SAP receiver for inbound stream discovery
26 // - Config format: same pipewire-aes67-instances.json (backward-compatible)
27
28 #if __has_include(<gst/gst.h>)
29 #define HAS_AES67_GSTREAMER
30
31 #include <gst/gst.h>
32
33 #include "fpphttp_types.h"
34
35 #include <atomic>
36 #include <chrono>
37 #include <cstdint>
38 #include <functional>
39 #include <map>
40 #include <mutex>
41 #include <string>
42 #include <thread>
43 #include <vector>
44
45 // ──────────────────────────────────────────────────────────────────────────────
46 // AES67 Protocol Constants (mirror fpp_aes67_common.py)
47 // ──────────────────────────────────────────────────────────────────────────────
48 namespace AES67 {
49
50 constexpr int RTP_PAYLOAD_TYPE = 96;
51 constexpr int AUDIO_RATE = 48000;
52 constexpr int AUDIO_RTP_TTL = 4;
53 // 1ms is mandatory for every AES67 device and is the only packet time Dante
54 // will transmit, so it is the interoperable default. It is also the only one
55 // that fits above stereo (see MaxPtimeForChannels), and measured better than
56 // 4ms here rather than worse: over 180k packets, worst-case transmission
57 // lateness 0.235ms against 0.64ms at 4ms ptime, because smaller buffers give
58 // the pacing thread less to slip. Existing configs keep whatever they have --
59 // this only applies where no ptime was ever stored.
60 constexpr int DEFAULT_PTIME_MS = 1; // 1ms packet time
61 constexpr int DEFAULT_PORT = 5004;
62 constexpr int DEFAULT_CHANNELS = 2;
63
64 // Warning raised when a pipeline will not start. WarningHolder::RemoveWarning()
65 // matches on the id *and* the exact message text, and these warnings have no
66 // timeout, so the strings have to be shared between the code that raises one
67 // and the code that retracts it -- otherwise a warning outlives the problem it
68 // describes and the banner only clears on an fppd restart.
69 constexpr int WARNING_ID_PIPELINE = 44;
70 constexpr const char* WARNING_SEND_FAILED = "AES67: audio send stream failed to start";
71 constexpr const char* WARNING_RECV_FAILED = "AES67: audio receive stream failed to start";
72
73 // Warning slot for the PTP daemon itself, kept separate from the pipeline one
74 // so a clock problem and a stream problem do not overwrite each other.
75 constexpr int WARNING_ID_PTP = 45;
76 constexpr const char* WARNING_PTP_NO_LOCK =
77 "AES67: PTP did not reach a usable lock before the streams started — "
78 "audio may be mistimed until the clock settles";
79
80 // The ways PTP can fail to come up at all. Every one of these describes
81 // something the user can fix without restarting fppd -- install linuxptp, free
82 // the interface, make the config path writable -- so every one of them has to
83 // be retractable, or the banner goes on naming a problem that has already been
84 // solved. InitPTP() retracts all four once ptp4l is confirmed running and
85 // ShutdownPTP() retracts them on the way down; a still-failing start raises
86 // them again.
87 constexpr const char* WARNING_PTP_NOT_INSTALLED =
88 "AES67: ptp4l not found — install the linuxptp package";
89 constexpr const char* WARNING_PTP_CONF_WRITE =
90 "AES67: could not write PTP configuration file";
91 constexpr const char* WARNING_PTP_SPAWN_FAILED =
92 "AES67: could not start the ptp4l clock-sync process";
93 constexpr const char* WARNING_PTP_START_FAILED =
94 "AES67: PTP clock sync (ptp4l) could not start on the configured interface";
95
96 // AES67 allows up to 8 channels per stream and FPP now carries all of them.
97 //
98 // This sat at 2 for a while because anything above stereo produced "Internal
99 // data stream error" and not one packet, which read as multichannel being
100 // unimplemented. It was one missing caps field: above stereo GStreamer needs
101 // an explicit channel-mask or negotiation fails (see ChannelMaskFor). The
102 // PipeWire side was already channel-aware -- the delay chain emits one delay
103 // node per channel with the right positions at any width.
104 //
105 // Verified end to end at 8 channels, 1ms ptime, with an 8-channel source:
106 // uniform 1152-byte payload, 1000 packets/s, all eight channels carrying
107 // audio, media clock within 0.1 ppm of PTP, jitter inside the 1ms AES67
108 // recommends.
109 //
110 // The stream's channel count and its output group's channel count are
111 // separate settings and both have to be set -- the group builds the delay
112 // chain that feeds this stream.
113 constexpr int MAX_SUPPORTED_CHANNELS = 8;
114
115 // Above this channel count pipewiresrc gets always-copy=true regardless of the
116 // sourceBufferCopy setting.
117 //
118 // Zero-copy hands out PipeWire's own buffers, and one is only returned to the
119 // pool once everything downstream has dropped its reference. audiobuffersplit
120 // fans a graph quantum into ~21 one-millisecond buffers, and past stereo it
121 // holds them long enough that the source misses cycles: buffers reach udpsink
122 // already past their PTS, so sync=true cannot pace what is already late and
123 // sends back-to-back to catch up. Measured, same box, same audio:
124 //
125 // channels payload back-to-back past the 1ms AES67 recommends
126 // 2 288B clean 0.0%
127 // 4 576B clean 0.0%
128 // 6 864B 7% 5.6%
129 // 8 1152B 32-50% 90.4%
130 //
131 // A deeper pool does not substitute: sourceMinBuffers=64 on its own still left
132 // 64% back-to-back. The copy does -- at 8 channels it restores clean pacing,
133 // jitter inside the 1ms recommendation, and the full presentation lead.
134 //
135 // Stereo keeps the zero-copy path, which is what the long soaks and the
136 // hardware testing to date actually ran. The copy is ~1.5MB/s at 8 channels.
137 constexpr int MULTICHANNEL_NEEDS_BUFFER_COPY = 2;
138 constexpr int DEFAULT_LATENCY_MS = 10;
139
140 // DSCP codepoints (AES67-2018 / AES-R16 QoS recommendations)
141 constexpr int AUDIO_DSCP = 34; // AF41 -- RTP audio (udpsink qos-dscp)
142 constexpr int PTP_DSCP = 46; // EF -- PTP event/general messages (ptp4l dscp_event/dscp_general)
143
144 // SCHED_FIFO priority for the thread that paces RTP packets onto the wire.
145 //
146 // Left at SCHED_OTHER nice 0 this thread misses a timeslice every few
147 // thousand packets, and each miss is a packet transmitted up to a full ptime
148 // late. Measured over 60k packets at 4ms ptime: worst-case lateness 8.6ms
149 // and 0.08% of packets past the 1ms AES67 recommends, versus 0.64ms worst
150 // case and none past 1ms once the thread is SCHED_FIFO. The load average
151 // was 0.24 on 4 cores in both runs -- this is wakeup latency, not
152 // contention, so no amount of spare CPU fixes it.
153 //
154 // 80 sits deliberately below the 88 PipeWire gives its data-loop: if both
155 // are ever runnable at once the audio graph must win, because starving it
156 // underruns every stream on the box, not just this one. Kernel RT
157 // throttling (sched_rt_runtime_us, 95% by default) is the backstop.
158 constexpr int SINK_RT_PRIORITY = 80;
159
160 // How much audio FlushSendPipelines() discards at a track boundary. Zero:
161 // it discards nothing, and the mechanism is kept only so this can be raised
162 // if some source ever does need it.
163 //
164 // The flush existed to throw away "stale audio queued between the old track
165 // stopping and the new one starting", but pipewiresrc is a live source -- it
166 // delivers the graph in real time and holds no backlog to discard. There was
167 // never anything stale there, so the drop only ever cut a hole in a stream
168 // that was already continuous, and everything downstream was machinery built
169 // to survive that hole: the drift probe reported it as a "source gap", the
170 // silence fill patched the timeline over it, and the sink queue had to be
171 // deep enough to cover it or a receiver heard the wire stop.
172 //
173 // Measured across track transitions, same box, same audio:
174 //
175 // flush source gaps worst wire gap worst tx jitter
176 // 213ms 213-232ms 157.7ms 156.7ms
177 // 50ms 23.2ms 6.13ms 5.18ms
178 // 0ms none (54 1.14ms 0.172ms
179 // transitions)
180 //
181 // At zero a transition is indistinguishable from steady state and clears the
182 // 1ms AES67 recommends, not just the 17ms it requires. It is also the only
183 // setting where AES67 carries exactly what the local sound card carries, so
184 // any artefact at a boundary is FPP's own audio rather than something this
185 // path invented.
186 constexpr int SOURCE_FLUSH_MS = 0;
187
188 // PTP (IEEE 1588) profile defaults
189 constexpr int DEFAULT_PTP_DOMAIN = 0;
190
191 // BMCA priority1 values selected by the "ptpRole" setting. Most professional
192 // AES67 gear ships priority1=128, so an FPP box left at 128 ties on priority
193 // and wins on clock identity (lowest MAC) -- silently hijacking the clock
194 // domain of a console or DSP. See issue #2848.
195 constexpr int PTP_PRIORITY_PREFER_MASTER = 127; // deliberately beat the usual 128
196 constexpr int PTP_PRIORITY_AUTO = 248; // yield to real gear; still GM when alone
197
198 // How long the SAP announcer re-checks the grandmaster every second after
199 // startup, to catch BMCA settling rather than waiting a whole announce cycle.
200 constexpr int PTP_CONVERGENCE_WINDOW_S = 60;
201
202 // TTL for the cached pmc query result. /aes67/status is HTTP-facing and each
203 // query forks a pmc process, so repeated hits must not fork per request.
204 constexpr int PTP_QUERY_CACHE_MS = 1000;
205
206 // ─── Waiting for the PTP clock before anchoring anything to it ──────────────
207 //
208 // A send pipeline is anchored to PTP time exactly once, when it is built:
209 // rtpL24pay latches "timestamp-offset" on the segment, so the RTP timeline
210 // cannot be re-anchored in place afterwards (setting it from a probe on the
211 // first buffer was measured to do nothing at all). That makes it a hard
212 // requirement that the clock read at build time is the clock the stream will
213 // keep running on.
214 //
215 // On a cold boot it was not. ptp4l was launched, confirmed alive 500ms later,
216 // and PTP declared "initialized" -- and GetOrCreateMediaClock() only proves
217 // the PHC can be *read*, which says nothing about whether it has been
218 // disciplined. As a follower it has not: the PHC free-runs from whatever the
219 // NIC came up with, and ptp4l steps it to grandmaster time seconds later, on
220 // entering SLAVE. Everything anchored in between is anchored to a time base
221 // that is about to jump.
222 //
223 // Reported on issue #2848 from a Pi 5 into a Yamaha MRX7-D, with the whole
224 // sequence in the log:
225 //
226 // 11:34:37.306 PTP initialized -- media clock using PHC /dev/ptp0
227 // 11:34:37.340 stream 1 RTP timeline anchored (timestamp-offset 3260165294)
228 // 11:34:39 selected best master clock 001dc1.fffe.149b4a
229 // 11:34:40 UNCALIBRATED -> SLAVE, then a first servo sample of
230 // rms 1265019998642539520 -- the pre-step offset
231 // 11:34:41 rms 1030ns <- the PHC has now moved under the anchor
232 //
233 // The grandmaster's epoch was ~6.81 hours ahead of the Pi's PHC, so the four
234 // streams declared a playout time 24,472 seconds in the past. They were not
235 // silent, which is what made it hard to see: aes67_verify measured 40.8
236 // packets/s in correctly-spaced bursts separated by 6.5 second holes, a media
237 // clock of 1857.83 samples/s (-961,298 ppm), and Dante red -- while
238 // /aes67/status still reported 4/4 pipelines running and PTP synced. The
239 // watchdog rebuilt the stack 90 seconds later and everything was correct on
240 // the first try, because by then the PHC was already disciplined.
241 //
242 // So: wait for the clock to settle before building anything on it.
243 //
244 // SLAVE alone is not the signal. The log above shows the step arriving
245 // *after* the transition, so a wait that stopped at SLAVE would still race it.
246 // The offset from master is the signal, and reaching it means the step has
247 // already happened.
248 constexpr int64_t PTP_LOCK_OFFSET_NS = 1000000; // 1ms
249
250 // How long InitPTP() will wait for that. This blocks fppd's startup, so it is
251 // deliberately short: it only runs on a box with AES67 enabled *and* PTP on,
252 // and the measured lock on the reference hardware is 3.2s to SLAVE and ~4s to
253 // microsecond offsets, which this covers with margin. Timing out is not a
254 // failure path -- the streams still start, and the step detector below picks
255 // up a clock that locks later (a grandmaster that boots slower than the Pi is
256 // an ordinary thing at a commissioning). Set "ptpLockWaitMs" to 0 in
257 // pipewire-aes67-instances.json to skip the wait entirely.
258 constexpr int PTP_LOCK_WAIT_MS = 5000;
259
260 // Poll cadence for that wait. Each poll forks two pmc queries, so 250ms is
261 // 40 forks across the whole 5s cap in the worst case, and none at all once the
262 // clock settles. PTP_QUERY_CACHE_MS does not help here -- the wait needs the
263 // live answer, not a second-old one -- so it queries with force.
264 constexpr int PTP_LOCK_POLL_MS = 250;
265
266 // The step detector, for a clock that moves *after* the pipelines were built:
267 // a grandmaster that appears late, or a BMCA changeover to a device on a
268 // different epoch. Both leave the anchor as wrong as the cold-boot race did,
269 // and neither can be fixed in place -- so the watchdog rebuilds.
270 //
271 // Measured as the change in (PTP - CLOCK_MONOTONIC) between two watchdog
272 // checks, which is the quantity the anchor actually depends on, so the test is
273 // "did the anchor become wrong" rather than a proxy for it. 100ms sits in a
274 // wide gap. Below it: a free-running PHC against the Pi's own crystal, which
275 // is the worst case since a disciplined one moves less -- measured on a Pi 5,
276 // -1.581ms across a 30 second check interval (52.7ppm, in line with the ~56ppm
277 // the drift loop corrects), so a 63x margin. Above it: ptp4l is configured to
278 // step anything at or above 1 second and slew everything below, so every step
279 // it can make trips this.
280 constexpr int64_t PTP_STEP_DETECT_NS = 100000000; // 100ms
281
282 // PTP Announce listener. No PTP management response carries the
283 // grandmaster's IP address -- TIME_STATUS_NP names it only by clock identity
284 // -- so the address is read off the source of the Announce messages ptp4l is
285 // already following, on the same multicast group and general-message port.
286 constexpr const char* PTP_MCAST_ADDRESS = "224.0.1.129";
287 constexpr int PTP_GENERAL_PORT = 320;
288
289 // An Announce arrives every 2s (logAnnounceInterval 1) and ptp4l declares the
290 // master gone after 3 missed ones, so an address older than this is no longer
291 // the live path to the clock and must not be reported as if it were.
292 constexpr int PTP_ANNOUNCE_STALE_MS = 10000;
293
294 // Anything on the LAN can send Announce and SAP packets, and each one with a
295 // new identity used to add an entry that only aged out minutes later -- a
296 // flood of made-up ones grew both maps until the board ran out of memory.
297 // A real domain has a handful of masters and a real network a few dozen
298 // streams; past these caps the entry heard from longest ago makes room.
299 constexpr size_t MAX_PTP_ANNOUNCE_SOURCES = 32;
300 constexpr size_t MAX_SAP_DISCOVERED = 256;
301 // SDP fields kept from an announcement, which can be up to a datagram long.
302 constexpr size_t MAX_SAP_FIELD_LEN = 256;
303
304 constexpr const char* DEFAULT_MULTICAST_IP = "239.69.0.1";
305 constexpr const char* AUDIO_FORMAT = "S24BE";
306 constexpr const char* SAP_MCAST_ADDRESS = "239.255.255.255";
307 constexpr int SAP_PORT = 9875;
308 constexpr int SAP_VERSION = 1;
309 constexpr int SAP_ANNOUNCE_INTERVAL_S = 30;
310 constexpr int SAP_TTL = 255;
311
312 // Largest RTP packet we will emit. AES67 forbids IP fragmentation, so one
313 // ptime of audio has to fit in a single datagram: 1500 (Ethernet MTU) - 20
314 // (IPv4) - 8 (UDP) = 1472 bytes of RTP. 1440 keeps headroom for the RTP
315 // header and for a VLAN tag or tunnel on the path. GStreamer's payloader
316 // defaults to 1400 and SPLITS anything larger, which silently breaks the
317 // min-ptime=max-ptime contract and emits packets that do not match the
318 // a=ptime we advertise -- so we always set this explicitly.
319 constexpr int MAX_RTP_PACKET_BYTES = 1440;
320
321 // L24 is 3 bytes per sample per channel.
322 188 inline int L24PayloadBytes(int ptime, int channels) {
323 188 return ptime * (AUDIO_RATE / 1000) * channels * 3;
324 }
325
326 // GStreamer channel mask for an AES67 channel count.
327 //
328 // Above stereo, caps without a channel-mask do not negotiate: audioconvert
329 // warns "Upstream caps contain no channel mask" and pipewiresrc's streaming
330 // thread stops with not-negotiated (-4), surfacing only as "Internal data
331 // stream error" on the bus and not one packet on the wire. Mono and stereo
332 // carry an implicit layout, which is why stereo worked and nothing above it
333 // did -- and why this looked like multichannel being unimplemented rather
334 // than one missing caps field.
335 //
336 // Bits are GstAudioChannelPosition values, and the layouts match what
337 // FPPINIT_Audio emits for the delay chain (FL FR / FL FR RL RR / 5.1 / 7.1).
338 169 inline guint64 ChannelMaskFor(int channels) {
339
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 161 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 8 times.
✗ Branch 5 not taken.
169 switch (channels) {
340 ✗ case 1: return 0x1; // FL (mono is positionless, but harmless)
341 161 case 2: return 0x3; // FL FR
342 ✗ case 4: return 0x33; // FL FR RL RR
343 ✗ case 6: return 0x3F; // FL FR FC LFE RL RR
344 8 case 8: return 0xC3F; // FL FR FC LFE RL RR SL SR
345 ✗ default: return 0; // unknown: omit the field
346 }
347 }
348
349 // Valid AES67 ptimes
350
3/4
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 182 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
185 inline bool IsValidPtime(int ptime) { return ptime == 1 || ptime == 4; }
351
352 // The largest valid ptime whose payload still fits in one datagram at this
353 // channel count. At 8 channels, 4ms of L24 is 4608 bytes -- over three
354 // times the limit -- so anything above stereo is 1ms only:
355 //
356 // ch 1ms 4ms
357 // 2 288 1152
358 // 4 576 2304 (too big)
359 // 6 864 3456 (too big)
360 // 8 1152 4608 (too big)
361 //
362 // 1ms is also the packet time Dante requires when receiving an AES67 flow,
363 // so it is the safe choice for interoperability as well as for MTU.
364 185 inline int MaxPtimeForChannels(int channels) {
365
2/2
✓ Branch 0 taken 177 times.
✓ Branch 1 taken 8 times.
185 return (L24PayloadBytes(4, channels) + 12 <= MAX_RTP_PACKET_BYTES) ? 4 : 1;
366 }
367
368 // Channel position names for SDP
369 const char* GetSDPChannelNames(int channels);
370
371 } // namespace AES67
372
373 // ──────────────────────────────────────────────────────────────────────────────
374 // Instance configuration — mirrors the JSON schema
375 // ──────────────────────────────────────────────────────────────────────────────
376 struct AES67Instance {
377 int id = 0;
378 std::string name;
379 bool enabled = true;
380 std::string mode = "send"; // "send" | "receive" | "both"
381 std::string multicastIP = AES67::DEFAULT_MULTICAST_IP;
382 int port = AES67::DEFAULT_PORT;
383 int channels = AES67::DEFAULT_CHANNELS;
384 std::string interface; // network interface (e.g., "eth0")
385 std::string sessionName;
386 int latency = AES67::DEFAULT_LATENCY_MS;
387 bool sapEnabled = true;
388 int ptime = AES67::DEFAULT_PTIME_MS;
389 };
390
391 struct AES67Config {
392 std::vector<AES67Instance> instances;
393 bool ptpEnabled = true;
394 std::string ptpInterface = "eth0";
395 int ptpDomain = AES67::DEFAULT_PTP_DOMAIN;
396
397 // BMCA participation:
398 // "auto" (default) -- priority1 248: still becomes grandmaster when
399 // it is the only clock on the domain, but yields
400 // to gear that wants the role.
401 // "follower" -- ptp4l slaveOnly: never becomes grandmaster.
402 // "master" -- priority1 127: prefer to win the election.
403 std::string ptpRole = "auto";
404
405 // How long InitPTP() waits for ptp4l to reach a settled clock before the
406 // send pipelines are anchored to it. See AES67::PTP_LOCK_WAIT_MS for why
407 // the wait exists and why it is short; 0 skips it, which restores the
408 // pre-#2848 behaviour and is only useful for reproducing that failure.
409 int ptpLockWaitMs = AES67::PTP_LOCK_WAIT_MS;
410
411 // Hold a sender idle until something in the audio graph actually feeds it.
412 //
413 // A send instance is a PipeWire *sink*: its pipewiresrc is created with
414 // node.autoconnect=false, so nothing links into it unless an Audio Output
415 // Group member names it as node.target. With nothing linked the pipeline
416 // cannot preroll, and gst_element_set_state() sits there for 30 seconds
417 // before returning FAILURE -- per instance, on every Apply. That is the
418 // chicken and egg a new instance walks straight into: it cannot be added
419 // to a group until it has been saved, and saving it costs half a minute
420 // of dead UI and a "stream failed to start" warning for a stream the user
421 // has not finished configuring yet.
422 //
423 // So when nothing feeds it, do not start it at all: the instance stays
424 // configured, ApplyConfig() returns promptly, GetStatus() reports it as
425 // waiting rather than failed, and the next apply -- which the Audio Output
426 // Groups page performs anyway, by restarting the stack -- starts it for
427 // real. See GraphFeedsSendNode().
428 //
429 // Set false in pipewire-aes67-instances.json for a graph FPP did not
430 // generate (a hand-written PipeWire conf, or nodes linked by hand with
431 // pw-link), where the group config cannot answer the question.
432 bool requireGroupSource = true;
433
434 // Diagnostic switches for the AES67 send path. Both default on; set
435 // either to false in pipewire-aes67-instances.json and hit Apply to take
436 // that change out of the pipeline without rebuilding, which is what makes
437 // a sender problem bisectable against real receiver hardware.
438 //
439 // ptpMediaClock -- drive the pipeline from PTP time and derive the RTP
440 // timestamps from it (AES67 "a=mediaclk:direct=0").
441 // sourcePacing -- ask PipeWire for a packet-sized quantum so RTP leaves
442 // evenly instead of one graph quantum at a time.
443 bool ptpMediaClock = true;
444 // Default OFF, and do not turn it on without solving what follows.
445 //
446 // It does fix the transmit cadence -- no gaps over 8ms in a 500-packet
447 // capture, and a Dante receiver's subscription state goes green -- but the
448 // audio is severely distorted, on both a 44100 and a genuine 48000 graph
449 // (confirmed on Yamaha MRX7-D hardware, issue #2848). The fractional
450 // quantum theory (192/48000 being 176.4 samples at 44100) is therefore
451 // wrong: it distorts just as badly where the request divides exactly.
452 //
453 // The cause is still unknown. It is not packet loss or timeline damage:
454 // sequence is unbroken, timestamp increments are exact, and the payload is
455 // statistically normal music with no splices at packet boundaries. The
456 // corruption is inside the packets, which is why no packet-level metric
457 // catches it.
458 //
459 // Note the priority this deserves: the bursty default path plays cleanly on
460 // real hardware even at the MRX7-D's minimum 0.25ms receive latency, so the
461 // cadence this would fix is not currently blocking anything.
462 //
463 // Prefer sinkPacing below, which fixes the same cadence without touching
464 // the graph and measurably does not distort. The distortion this one
465 // causes is now measurable too -- scripts/aes67_verify decodes the RTP
466 // payload and correlates it against the source file, which is the check
467 // that was missing when the note above was written.
468 bool sourcePacing = false;
469
470 // Pace the packets at the sink instead of at the source.
471 //
472 // The default path bursts: pipewiresrc hands over one graph quantum at a
473 // time and udpsink (sync=false) puts the whole lot on the wire in
474 // microseconds. Measured on a 1024-sample/44100 graph that is 5.8 packets
475 // back-to-back every 23.22ms -- 83% of packets arrive under 1ms after the
476 // one before, then nothing for 23ms.
477 //
478 // Which receivers that breaks depends entirely on how they clock playout.
479 // Hardware that places samples by RTP timestamp against its own PTP time
480 // does not care when a packet arrived, only that it arrived before its
481 // deadline -- which is why a Yamaha MRX7-D plays this cleanly even at
482 // 0.25ms. Software monitors generally play out by arrival into a small
483 // jitter buffer, so a 23ms silence underruns them every quantum and the
484 // audio fragments. Both reports on issue #2848 are the same defect seen
485 // from those two sides.
486 //
487 // sourcePacing fixes the cadence by shrinking the quantum, but it shrinks
488 // it for the whole graph -- the sound card included -- and the audio comes
489 // out distorted. This does not touch the graph: the queue below gives the
490 // sink its own thread, so it can wait for each packet's running time
491 // without the backpressure reaching the live source. That backpressure is
492 // what cost ~36% of the stream the last time sync=true was tried without a
493 // queue in between.
494 //
495 // sinkPacingMs is how long each packet is held back so that it is still in
496 // the future when the sink gets it. It must exceed one graph quantum
497 // (23.22ms at 1024/44100) or every buffer arrives late and goes out
498 // immediately, which is the burst all over again. 40ms leaves real margin
499 // on top of that on purpose: too small a value does not fail loudly, it
500 // silently reverts to bursting, which is expensive to diagnose. It costs
501 // sender latency, but the receiver no longer has to absorb the burst as
502 // link offset.
503 //
504 // Measured on a 1024/44100 graph, 4ms ptime, against the source audio:
505 //
506 // default path sinkPacing
507 // median gap 0.028ms 3.999ms
508 // packets per burst 5.8 1.00
509 // back-to-back 83% 0.0%
510 // content vs source 0.995 correlation 1.000 correlation
511 //
512 // Unlike sourcePacing it does not touch the graph, so it does not distort.
513 //
514 // It is NOT yet a fix on its own, and default OFF for a concrete reason:
515 // the queue drains. The sink releases packets on the PTP clock while the
516 // audio arrives on the sound card crystal, and on this hardware those are
517 // 56ppm apart (see adaptiveResample). Starting from a 40-60ms queue that
518 // is 12-18 minutes to empty, which matches what was measured: cadence held
519 // at exactly 4.000ms and 250.2 packets/s for ~15 minutes, then the queue
520 // touched 0 and in that same instant the median gap went to 0.031ms, 60%
521 // of packets went back-to-back and the rate fell to 187.6/s. It does not
522 // recover -- with nothing buffered every packet arrives past its send time
523 // and udpsink dumps them all immediately.
524 //
525 // So sink pacing is correct but not self-sufficient: it needs the audio
526 // rate locked to the PTP clock, i.e. a working adaptiveResample, or the
527 // queue empties and it fails worse than not pacing at all. Raising
528 // sinkPacingMs only buys time proportionally; it does not fix a systematic
529 // rate difference.
530 //
531 // Verified with scripts/aes67_verify, not yet against Dante or Yamaha
532 // hardware.
533 // Default ON since 2026-09: this is the validated configuration, proven on
534 // a Pi 5 against a Yamaha MRX7-D / Brooklyn II grandmaster (2.5h and 5h50m
535 // continuous, follower mode, Dante Controller green) as well as a 4h soak
536 // here. The four are not independent -- see the incoherence warnings in
537 // ParseConfig -- so they move together. Only AES67 send pipelines are
538 // affected; a box with no AES67 instance sees no change.
539 bool sinkPacing = true;
540 int sinkPacingMs = 40;
541
542 // Reconcile the audio clock with the PTP clock by inserting or dropping
543 // samples, instead of by varying a resampler's rate.
544 //
545 // The drift is a fixed ~56ppm between the sound card's crystal and the
546 // NIC's PHC (see adaptiveResample), and both variable-rate elements that
547 // could correct it have been measured and rejected -- "speed" and "pitch"
548 // each accept a rate property and neither applies it to a live stream.
549 //
550 // "audiorate" attacks it from the other side. It does not resample: it
551 // compares each buffer's timestamp against the sample count it has already
552 // emitted and inserts or drops samples to make the two agree. Since
553 // pipewiresrc timestamps buffers on the pipeline clock (the PHC) while the
554 // sample count comes from the card, that difference IS the drift, so the
555 // correction needs no control loop, no gains and no dead time -- the thing
556 // that made the previous attempt oscillate.
557 //
558 // It also settles the actuation question directly: "add" and "drop" are
559 // readable sample counters, so whether it is doing anything is a fact
560 // rather than an inference from a property read-back. At 56ppm expect
561 // ~2.7 samples/s of correction.
562 //
563 // rateMatchToleranceNs is how far the two may diverge before it acts. 0
564 // corrects continuously; the element's own default of 40ms lets the error
565 // build and then dumps a 40ms correction, which is plainly audible.
566 //
567 // MEASURED, AND IT DOES NOT WORK. Keep it default OFF. audiorate is the
568 // third rejected actuator, after "speed" and "pitch", and it fails
569 // differently from both -- it actuates, but wrongly, and then unrecoverably:
570 //
571 // - It over-corrects by more than an order of magnitude. Steady state it
572 // inserted ~40 samples/s (833ppm) where 2.7 samples/s (56ppm) was
573 // needed. It is reconciling per-buffer timestamp rounding, not drift,
574 // and no tolerance from 0 to 5ms changed that.
575 // - After ~11 minutes it enters permanent runaway, inserting ~12,000
576 // samples/s -- a quarter of the stream becomes fabricated silence, and
577 // it never recovers. Reproduced twice, at 10-12 and 11 minutes, on an
578 // otherwise healthy system. The shape fits a single forward jump in
579 // the timeline leaving a deficit it then chases forever.
580 //
581 // What this experiment DID establish, and the reason the switch is kept:
582 // with the rate corrected, sinkPacing's queue stops draining. Over 28
583 // minutes the cadence held at 4.00ms with uniform 1152-byte payloads and
584 // 250.0 packets/s, and the queue oscillated 56-80ms with no downward trend,
585 // against a collapse at ~15 minutes without it. So "correct the sample
586 // rate, then pace at the sink" is the right architecture; what is missing
587 // is only a correct rate corrector.
588 //
589 // That corrector needs to (a) apply a slow, near-constant ~56ppm
590 // correction rather than tracking per-buffer jitter, (b) preserve exact
591 // ptime blocking, and (c) reset rather than accumulate across a timeline
592 // discontinuity. No stock GStreamer element on the Pi image does all
593 // three, so it likely has to be written -- a fractional-delay resampler
594 // with its own read pointer, which keeps every output buffer exactly one
595 // packet long while consuming slightly more or fewer input samples.
596 bool rateMatch = false;
597 guint64 rateMatchToleranceNs = 0;
598
599 // Correct the sound card / PHC rate difference with libsamplerate, which
600 // is what the three rejected elements above were each trying to be.
601 //
602 // See the DriftResampleProbe comment in AES67Manager.cpp for the control
603 // law. In short: the ratio is computed directly from (input frames / PHC
604 // seconds) rather than servoed towards a target, because that quantity is
605 // the difference between two crystals and is therefore near-constant --
606 // which is why this does not oscillate the way the earlier loops did.
607 //
608 // Requires libsamplerate at build time; without it the switch does nothing
609 // and the send path behaves exactly as before.
610 //
611 // MEASURED. The resampler itself is transparent and the correction is
612 // right, but this is default OFF because it does not survive past ~11
613 // minutes -- see below.
614 //
615 // Transparency, measured warp-insensitively by comparing power spectra
616 // against the source file (a 57ppm warp moves a bin by 0.006%, so a PSD
617 // sees straight through it): OFF gives mean -3.01 dB / stdev 0.00 dB, ON
618 // gives mean -3.01 dB / stdev 0.09 dB. Indistinguishable.
619 //
620 // Do NOT judge this with correlation against the source: the stream is
621 // deliberately time-warped, so a fixed alignment cannot match a whole
622 // window and correlation reads ~0.93 on audio that is provably clean. It
623 // recovers as the window shrinks -- 0.42 at 2s, 0.85 at 100ms, tracking
624 // intra-window slip -- which is a property of the measurement, not the
625 // audio.
626 //
627 // Correction: trim converges to +57ppm and holds, leaving ~2.5ppm residual
628 // against 56ppm uncorrected, with the alignment slipping exactly -5.5
629 // samples per 2s as intended.
630 //
631 // With sinkPacing this produced the first fully correct AES67 stream in
632 // this file's history: 4.000ms cadence, 1.00 packets per burst, 250.1
633 // packets/s, uniform 1152-byte payloads, pacing queue stable at 56-80ms.
634 // That held for 9-11 minutes and then degraded -- send rate falling to
635 // 155/s with ragged payloads, and the media timeline stepping by +/-23.2ms,
636 // which is exactly one graph quantum at 1024/44100.
637 //
638 // That degradation is NOT this code. The same ~11 minute onset appeared
639 // with audiorate, and sinkPacing alone collapsed at ~15 minutes; the media
640 // file is 220 minutes long and playback was never restarted, so it is
641 // neither a track boundary nor a restart. Something starves the AES67
642 // branch after ten-odd minutes of playback and that is the next thing to
643 // find -- it is very likely the original "sounds fine then goes fragmented"
644 // report on #2848, which every fix so far has merely postponed.
645 // Default ON since 2026-09: this is the validated configuration, proven on
646 // a Pi 5 against a Yamaha MRX7-D / Brooklyn II grandmaster (2.5h and 5h50m
647 // continuous, follower mode, Dante Controller green) as well as a 4h soak
648 // here. The four are not independent -- see the incoherence warnings in
649 // ParseConfig -- so they move together. Only AES67 send pipelines are
650 // affected; a box with no AES67 instance sees no change.
651 bool driftResample = true;
652
653 // Count bytes in at pipewiresrc and out at udpsink, and log both rates.
654 //
655 // Diagnostic for the send rate collapsing partway through a long playback
656 // (250/s to 188/s at ~11 minutes, then to 151/s at ~19 minutes, with
657 // PipeWire reporting no xruns and CPU, memory, thread and fd counts all
658 // flat). Since PipeWire says it is delivering, the audio is going missing
659 // between pipewiresrc and udpsink, and these two counters say which side
660 // of the pipeline loses it.
661 bool pipelineStats = false;
662
663 // Take audio from PipeWire at the graph's own rate and convert to the AES67
664 // rate inside GStreamer, instead of asking PipeWire to deliver 48kHz from a
665 // 44.1kHz graph.
666 //
667 // This is the fault behind "AES67 sounds fine, then goes fragmented". With
668 // a 44100 graph, pipewiresrc negotiates 48000 and PipeWire resamples for
669 // that node alone; measured at the pipewiresrc pad, delivery holds at
670 // 287,968 B/s for 13 minutes, then steps to exactly 0.750x, then to 0.600x,
671 // and stays there. PipeWire reports no xruns for the node throughout, and
672 // CPU, memory, threads, fds, temperature and the graph quantum are all flat
673 // across the whole run.
674 //
675 // Our own pipeline is not involved: byte counters at pipewiresrc and
676 // udpsink track each other at a constant 1.010 ratio the entire time, so
677 // nothing downstream loses a sample -- there is simply less audio arriving.
678 // The physical sinks are unaffected because they run at the graph rate and
679 // are never resampled, which is exactly why the sound card stays clean
680 // while the AES67 stream falls apart.
681 //
682 // Default OFF: this was written as a candidate fix for the collapse and it
683 // is not one -- delivery degrades identically whether the source is taken
684 // at 48kHz or at the graph's own rate. PipeWire's per-node resampling is
685 // not at fault either way; pw-record's own 48k-from-44.1k stream runs at
686 // 99.9% of realtime indefinitely. Kept as a knob, not a fix.
687 bool nativeSourceRate = false;
688
689 // Copy each buffer out of PipeWire's pool instead of referencing it, and
690 // negotiate a deeper pool.
691 //
692 // Candidate fix for the send rate collapsing partway through playback.
693 // Delivery to pipewiresrc steps to exactly 3/4 then 3/5 of nominal and
694 // stays there, with no xruns on the node and every resource metric flat --
695 // the shape of PipeWire skipping cycles because no buffer is free to fill.
696 // pipewiresrc negotiates min-buffers=2 and, with always-copy off, a buffer
697 // is only returned to the pool once everything downstream has dropped its
698 // reference, so anything that holds one starves the next cycle.
699 //
700 // Ruled out first, so this is not a guess at the level above: the graph is
701 // healthy (recording the sound card's monitor gives 99.9% of realtime while
702 // AES67 is at 52%), our own pipeline loses nothing (byte counters at
703 // pipewiresrc and udpsink hold a constant 1.010 ratio), and it is not
704 // per-node resampling (it degrades identically at the graph's native rate,
705 // and pw-record's own 48k-from-44.1k stream is unaffected).
706 // THIS is the fix. Isolated by A/B: a 16 buffer pool on its own holds the
707 // send rate flat for 26 minutes, with always-copy off and the source still
708 // taken at 48kHz. So the copy is not needed -- the pool was simply too
709 // shallow to absorb normal downstream reference-holding, and PipeWire
710 // skipped the cycles it could not fill.
711 //
712 // always-copy remains available for the case where something downstream
713 // genuinely retains buffers, at the cost of a memcpy per buffer.
714 //
715 // Above stereo that case is real, so the pipeline turns the copy on by
716 // itself -- see MULTICHANNEL_NEEDS_BUFFER_COPY. Setting this true forces
717 // it on at any channel count; it is never forced off.
718 bool sourceBufferCopy = false;
719 int sourceMinBuffers = 16;
720
721 // Schedule the capture node from PTP instead of from the sound card.
722 //
723 // This is the producer-side half of the pool problem. pipewiresrc receives
724 // into sourceMinBuffers slots that PipeWire fills at the graph's rate --
725 // the card crystal, ~54ppm slow of PTP here -- while the pipeline drains
726 // them against the PTP clock. driftResample corrects the rate downstream
727 // of the pool, so it cannot refill it; the pool empties after (N-1) quanta
728 // and the source then skips a cycle per rotation for good.
729 //
730 // PipeWire supports exactly this case. A support.node.driver clocked from
731 // the NIC's PHC drives any node that joins its node.group, and PipeWire
732 // rate-matches at the boundary with the card-driven graph using the
733 // adapter's DLL-driven adaptive resampler -- upstream of our pool rather
734 // than downstream of it. It is the same mechanism pipewire-aes67.conf
735 // uses, where the comment on aes67.driver-group calls it "force rate
736 // matching on the AES67 node rather than other nodes".
737 //
738 // Requires a driver node to exist with a matching group; FPP ships one in
739 // 98-fpp-ptp-driver.conf. Harmless if absent -- the node keeps the graph
740 // driver and behaves exactly as before.
741 // Run the pipeline on the graph's clock and use PTP only for the media
742 // clock and the RTP timestamps.
743 //
744 // MEASURED, AND IT IS THE WRONG ARCHITECTURE. Keep it default OFF.
745 //
746 // It does fix the source side -- the ring stops draining, because
747 // pipewiresrc no longer pulls on PTP time while PipeWire fills at the card's
748 // rate. But rtpL24pay derives RTP timestamps from the pipeline's running
749 // time, so moving the pipeline off PTP unanchors the whole RTP timeline:
750 // presentation timing measured -36,135,744 ms, and aes67_verify put the
751 // media clock at -56.3ppm, i.e. not locked at all. A ts-offset servo does
752 // not rescue that; the timestamps themselves would have to be re-derived.
753 //
754 // The deeper reason not to pursue it: it keeps the sound card as clock
755 // master and bolts PTP on beside it, and that is the opposite of how AES67
756 // is done. In professional practice the audio sample clock is slaved to
757 // PTP -- Dante and RAVENNA interfaces derive their word clock from the
758 // network. The Linux reference implementation (bondagit/aes67-linux-daemon)
759 // uses Merging's MergingRavennaALSA.ko, a kernel module presenting a virtual
760 // ALSA device that IS the PTP clock domain; everything else adapts to it.
761 // That is the same shape as PipeWire's node.group = pipewire.ptp0, which
762 // means the "graph takeover" that approach causes -- the sound cards moving
763 // onto the PTP driver and being rate-matched -- is not a side effect to be
764 // avoided. It is the intended behaviour, and the point of a clock domain.
765 // Default ON since 2026-09: this is the validated configuration, proven on
766 // a Pi 5 against a Yamaha MRX7-D / Brooklyn II grandmaster (2.5h and 5h50m
767 // continuous, follower mode, Dante Controller green) as well as a 4h soak
768 // here. The four are not independent -- see the incoherence warnings in
769 // ParseConfig -- so they move together. Only AES67 send pipelines are
770 // affected; a box with no AES67 instance sees no change.
771 bool splitClockDomains = true;
772
773 // Transmit lead the splitClockDomains servo holds, in milliseconds.
774 //
775 // How far ahead of its own declared playout time each packet goes on the
776 // wire. Bounded from both directions, and the window is narrower than it
777 // looks.
778 //
779 // Too small and the sender's own jitter eats the margin: a packet that
780 // slips past its playout time arrives late, and a conformant receiver
781 // discards it and plays silence -- that is the -23.7ms failure that
782 // silenced a Yamaha MRX7-D in issue #2848.
783 //
784 // Too large and the packet arrives before the receiver has anywhere to put
785 // it. The lead has to be held in the receiver's playout buffer, and that
786 // buffer is sized by the device's link offset -- so the ceiling is set by
787 // the smallest buffer on the network, not by anything here. The 20ms this
788 // used to default to was chosen against a Dante/RAVENNA link offset of
789 // 0.25-5ms "with a wide margin", which was the wrong reading: the margin is
790 // the buffer, and 20ms overruns it. It survived a Brooklyn II, which has
791 // the depth to absorb it, and failed on the Ultimo-class receivers that do
792 // not -- reported as late and missing packets, on a stream whose payload,
793 // sequence, timestamps and pacing all measured clean.
794 //
795 // 3ms sits inside both bounds and is verified on both: Ultimo and Brooklyn
796 // II receiving the same stream, where 20ms worked only on the Brooklyn. It
797 // still clears the sender's measured worst-case transmission lateness --
798 // 0.235ms at 1ms ptime over 180k packets, 0.64ms at 4ms -- by an order of
799 // magnitude. What it no longer clears is a pacing thread that failed to
800 // reach SCHED_FIFO, which can transmit a whole ptime late; that already
801 // warns at pipeline start, and now it matters.
802 //
803 // Global, not per-instance: one lead for every stream on the box. Raising
804 // it for a receiver with a deep buffer raises it for the shallow ones too.
805 int targetLeadMs = 3;
806
807 // Mix a permanent silence source in with the captured audio.
808 //
809 // The AES67 media clock is a sample count, so audio that never arrives puts
810 // the timeline permanently behind PTP -- it cannot be corrected later,
811 // because the samples are not late, they are absent. FPP tearing down and
812 // rebuilding its media pipeline at a track change leaves a ~232ms hole, and
813 // measured with a 25s file on repeat that ran the media clock to -6589ppm:
814 // AES67 drifting ahead of the local sound card by ~180ms per track, and
815 // accumulating. The listener hears tracks butted together on AES67 while
816 // the sound card has the pause.
817 //
818 // Silence is the additive identity, so mixing a constant silence source
819 // with the real audio needs no arbitration: silence alone when nothing is
820 // playing, the audio itself when something is. Proven by hand -- linking a
821 // silence source alongside the media took the media clock from -6589ppm to
822 // +0.9ppm. That was done in FPP's routing matrix and killed the audio, so
823 // it belongs here in our own pipeline instead, where nothing else is
824 // arbitrating the links.
825 // Default ON since 2026-09: this is the validated configuration, proven on
826 // a Pi 5 against a Yamaha MRX7-D / Brooklyn II grandmaster (2.5h and 5h50m
827 // continuous, follower mode, Dante Controller green) as well as a 4h soak
828 // here. The four are not independent -- see the incoherence warnings in
829 // ParseConfig -- so they move together. Only AES67 send pipelines are
830 // affected; a box with no AES67 instance sees no change.
831 bool sourceSilenceFloor = true;
832
833 bool sourcePtpGroup = false;
834 std::string sourcePtpGroupName = "pipewire.ptp0";
835
836 // Schedule the capture node asynchronously from its producer. Required
837 // whenever sourcePtpGroup is on: crossing driver domains without it stalls
838 // the stream outright. See the comment at the node.async call site.
839 bool sourceAsync = true;
840
841 // Adaptive resampling: continuously trim the send stream's rate so the
842 // media timeline advances at exactly PTP rate.
843 //
844 // Without it the audio is clocked by the sound card's crystal while the RTP
845 // timestamps advance on PTP, so the two walk apart -- measured at -111ppm,
846 // i.e. 6.7ms per minute. A receiver holding 5ms (the Yamaha MRX7-D's
847 // maximum) runs out in under a minute and mutes; at its 0.25ms minimum, in
848 // seconds. That is why AES67 audio plays cleanly right after a restart and
849 // then dies, which cost a lot of confusing test reports on #2848.
850 // Default OFF: the loop is implemented but NOT yet working. With the
851 // gains it shipped with it drove the stream to +890ppm (worse than the
852 // -111ppm it corrects); with the gains fixed it now makes no correction at
853 // all -- measured -112.6ppm against a -110.9ppm uncorrected baseline.
854 //
855 // Next step is to find out which half is broken, which needs the
856 // per-iteration numbers this loop already logs at debug level: either
857 // gst_element_query_position() is not reporting the post-"speed" media
858 // timeline (so `ratio` is always ~1 and no correction is ever computed), or
859 // the speed property is not taking effect on a running pipeline. Check
860 // `ratio` in the log first: if it tracks the real drift the measurement is
861 // fine and the element is at fault, and vice versa.
862 //
863 // UPDATE: the "speed" element is not a working actuator. Driven from 0 to
864 // the full -500ppm clamp on a live stereo stream, the measured drift moved
865 // by ~35ppm (-80 -> -45) where ~500ppm was asked for. It accepts the
866 // property and reads it back -- which is what earlier notes here wrongly
867 // took as proof of actuation -- but does not apply it to a live source,
868 // having been written for changing playback rate of seekable media.
869 //
870 // The measurement half of the loop IS sound: on a stereo stream it reads a
871 // steady -40 to -100ppm, consistent with the -110.9ppm measured
872 // independently from packet captures. So what needs replacing is the
873 // actuator, not the control loop.
874 //
875 // "pitch" (soundtouch) was the next candidate and it is also not usable:
876 //
877 // - It does not actuate either. Over six actuations the trim ramped
878 // 83ppm (+11.8 -> -71.4) while the measured ratio just wandered around
879 // -50ppm with +/-35ppm of noise -- no systematic response, the same
880 // signature "speed" gave.
881 // - It destroys the transmit cadence. soundtouch emits in large internal
882 // blocks, so everything downstream arrives late in clumps: with pitch in
883 // the chain, 16.05 packets per burst and 93.8% back-to-back with the
884 // pacing queue pinned at 0, against 1.00 packets and 0.0% with it
885 // removed (A/B on the same track, same build).
886 //
887 // Why this matters more than it looks: this is not a nice-to-have. The
888 // drift is two free-running crystals -- the NIC's PHC against the sound
889 // card's -- measured at -55.8ppm over 10s, -56.36ppm over 60s and -56.9ppm
890 // from packet captures, three independent methods agreeing to ~1ppm. When
891 // FPP is PTP master (as it is by default with no other grandmaster on the
892 // network) ptp4l never disciplines the PHC at all, so there is nothing to
893 // lock to and no PTP setting can remove this. It also drains the sinkPacing
894 // queue, so both the drift and the pacing failure are this one cause.
895 //
896 // What is still needed is a variable-rate resampler that (a) actually
897 // responds to a rate property on a live stream and (b) preserves buffer
898 // cadence. Neither "speed" nor "pitch" does both.
899 bool adaptiveResample = false;
900 };
901
902 // ──────────────────────────────────────────────────────────────────────────────
903 // Pipeline wrapper — one per send or receive direction of an instance
904 // ──────────────────────────────────────────────────────────────────────────────
905 struct AES67Pipeline {
906 int instanceId = 0;
907 bool isSend = true;
908 GstElement* pipeline = nullptr;
909 GstElement* rtpbin = nullptr;
910 GstBus* bus = nullptr; // polled by watchdog (no GLib main loop)
911
912 // Pipeline state
913 bool running = false;
914 std::string errorMessage;
915 // Posted an ERROR on its bus. The pipeline stays in PLAYING with its
916 // streaming stopped and says so only once, so the watchdog keeps this
917 // until the rebuild replaces the pipeline.
918 bool busError = false;
919
920 // Buffer-drop probe: while > 0, the probe drops buffers and subtracts
921 // each one's duration. Used by FlushSendPipelines() to discard stale
922 // audio between tracks.
923 //
924 // This is a DURATION, not a buffer count, and that distinction was the
925 // whole of the track-transition fault. The old code dropped a fixed 10
926 // buffers, sized against a 256-sample quantum ("~53ms" per its comment).
927 // Nothing runs a 256-sample quantum: at the 1024 both a Pi 5 and this box
928 // actually use, 10 buffers is 213ms at 48kHz and 232ms at 44.1kHz -- four
929 // times the audio it meant to discard. That self-inflicted gap is what
930 // the drift probe then reported as a "source gap" and filled with
931 // silence, and what a receiver saw as ~158ms of dead wire once the sink
932 // queue had drained through it. Both numbers matched to the millisecond
933 // on two different machines.
934 std::atomic<gint64> dropRemainingNs = 0;
935 GstPad* probePad = nullptr; // pad where probe is installed
936 gulong probeId = 0; // installed probe handle (0 = none)
937
938 // Zombie detection: track bytes pushed by udpsink across watchdog cycles.
939 // If bytes haven't increased for 2 consecutive checks while pipeline
940 // reports PLAYING, the pipewiresrc has lost its PipeWire connection.
941 uint64_t lastByteCount = 0;
942 int stallCount = 0; // consecutive checks with no progress
943 // Under-delivery detection. A stall is easy -- bytes stop entirely -- but
944 // the failure that actually bites is partial: after roughly 105-110
945 // minutes of pipeline uptime the send rate settles around 90% of nominal
946 // with periodic RTP timestamp gaps, and it never recovers on its own, so
947 // the stream plays broken until fppd is restarted. bytes-served keeps
948 // climbing throughout, so the stall check above cannot see it.
949 //
950 // The cause is the source buffer pool draining, and the interval is set by
951 // clock drift, not by uptime. pipewiresrc receives into sourceMinBuffers
952 // slots while the pipeline consumes against PTP; the PipeWire graph clock
953 // free-runs ~54ppm slow of the NIC's PHC on this hardware, so the pool
954 // loses that difference steadily and empties after (N-1) quanta of
955 // accumulated slip. From then on the source skips exactly one graph cycle
956 // every N buffers, forever, until the pipeline is rebuilt with a full pool.
957 //
958 // Measured both ways round, which is what confirmed it:
959 //
960 // min-buffers onset buffer onset gap spacing implied drift
961 // 16 273985 97.4m every 16 54.7 ppm
962 // 8 128604 45.7m every 8 54.4 ppm
963 //
964 // against 54.2ppm measured directly with phc_ctl. Onset ratio 2.130 vs
965 // 15/7 = 2.143 predicted. Forcing the graph to 48000 shifts the onset to
966 // 98 minutes from 108 -- 10% off in wall time, but the same buffer count,
967 // which is what first showed the trigger counts cycles rather than seconds.
968 //
969 // So sourceMinBuffers is a reservoir, not a fix: it sets how long the
970 // stream survives, not whether it degrades. Raising it buys proportional
971 // time and nothing else. A reporter whose clock drifts ~17ppm sees the
972 // same failure at 5h18m for the same reason.
973 //
974 // Earlier notes here read this as uptime-driven. That was fitted to runs
975 // that happened to contain no restarts; the cadence in fact continues
976 // straight through an fppd restart, which is what ruled uptime out.
977 std::chrono::steady_clock::time_point lastByteTime{};
978 int lowRateCount = 0;
979 int channels = AES67::DEFAULT_CHANNELS; // for the expected byte rate
980 int ptime = AES67::DEFAULT_PTIME_MS; // packets/s, for the header bytes in it
981 bool pacingTuned = false; // latency-derived shift applied yet?
982 };
983
984 // ──────────────────────────────────────────────────────────────────────────────
985 // SAP discovered stream (from remote announcements)
986 // ──────────────────────────────────────────────────────────────────────────────
987 struct SAPDiscoveredStream {
988 uint16_t msgIdHash = 0;
989 std::string originAddress;
990 std::string sessionName;
991 std::string multicastIP;
992 int port = 0;
993 int channels = 2;
994 int ptime = 4;
995 std::string ptpClockId;
996 uint64_t lastSeenMs = 0;
997 int autoCreatedInstanceId = -1; // ID of auto-created receive pipeline, or -1
998 };
999
1000 // ──────────────────────────────────────────────────────────────────────────────
1001 // AES67Manager — singleton managing all AES67 GStreamer pipelines
1002 // ──────────────────────────────────────────────────────────────────────────────
1003 class AES67Manager {
1004 public:
1005 static AES67Manager& INSTANCE;
1006
1007 // HTTP API endpoint (registered at /aes67)
1008 HttpResponsePtr render_GET(const HttpRequestPtr& req);
1009
1010 // Lifecycle
1011 bool Init(); // Called from fppd startup
1012 void Shutdown(); // Called from fppd shutdown
1013
1014 // Apply configuration — reads JSON and rebuilds pipelines
1015 // Called from PHP API (POST /api/pipewire/aes67/apply) and boot
1016 bool ApplyConfig();
1017
1018 // Cleanup — stop all pipelines, remove PipeWire configs
1019 void Cleanup();
1020
1021 // Status query — for PHP API (GET /api/pipewire/aes67/status)
1022 struct Status {
1023 struct PipelineStatus {
1024 int instanceId;
1025 std::string name;
1026 std::string mode; // "send" or "receive"
1027 bool running;
1028 std::string error;
1029 // Rate pipewiresrc actually negotiated with the graph, before our
1030 // resampler. 48000 means the audio reaches AES67 untouched; any
1031 // other value means we are rate-converting off the sound card's
1032 // clock. This is the number that matters for AES67 -- the graph's
1033 // default.clock.rate does not tell you what this stream is fed,
1034 // because per-card and per-group rates sit in between.
1035 int sourceRate = 0;
1036 // Configured and enabled, but deliberately not started because
1037 // nothing in the audio graph feeds it -- see
1038 // AES67Config::requireGroupSource. Not an error: `note` carries
1039 // the reason for the UI to render as guidance rather than a
1040 // failure.
1041 bool waitingForSource = false;
1042 std::string note;
1043 };
1044 std::vector<PipelineStatus> pipelines;
1045 bool ptpSynced = false;
1046 int64_t ptpOffsetNs = 0; // offset from PTP master
1047 // False when pmc gave no usable master_offset, so ptpOffsetNs is a
1048 // default rather than a measurement. Reported as a null offsetNs
1049 // rather than a zero that reads as a perfect lock.
1050 bool ptpOffsetValid = false;
1051 std::string ptpGrandmasterId;
1052 // IP the grandmaster's Announce messages come from -- our own
1053 // interface address when we hold the role. Empty when no Announce
1054 // for this identity has been seen recently (see PTP_ANNOUNCE_STALE_MS).
1055 std::string ptpGrandmasterAddress;
1056 // True when that address is the boundary clock relaying the domain
1057 // rather than the grandmaster itself, i.e. stepsRemoved > 0.
1058 bool ptpGrandmasterViaBoundary = false;
1059 std::string ptpPortState; // ptp4l portState (MASTER/SLAVE/LISTENING/...)
1060 bool ptpIsGrandmaster = false; // true when *we* hold the grandmaster role
1061 // False when the current pipelines anchored their RTP timelines to a
1062 // clock that had not settled, so the anchors may be wrong by however
1063 // far it has moved since -- the failure on #2848, where every other
1064 // field here read healthy while the wire was not. True when the
1065 // clock was settled, and also when nothing is anchored to PTP.
1066 bool ptpLockedAtStart = false;
1067 // Copied out of m_config under m_configMutex so the HTTP handler can
1068 // render them without touching shared config itself.
1069 bool ptpEnabled = false;
1070 int ptpDomain = 0;
1071 std::string ptpRole;
1072 // The graph's clock rate, for context only -- see PipelineStatus::sourceRate
1073 // for what actually feeds a given AES67 stream.
1074 int graphSampleRate = 0;
1075 std::vector<SAPDiscoveredStream> discoveredStreams;
1076 };
1077 Status GetStatus();
1078
1079 // Self-test — validates AES67 subsystem components.
1080 // Returns JSON with pass/fail for each test.
1081 struct TestResult {
1082 std::string testName;
1083 bool passed;
1084 std::string message;
1085 };
1086 std::vector<TestResult> RunSelfTest();
1087
1088 // Check if manager is active
1089 4054 bool IsActive() const { return m_active.load(); }
1090
1091 // A zero-hop RTP branch on a media pipeline's tee. The branches are
1092 // disabled (see GStreamerOutput::AttachAES67Branches()); the type stays
1093 // for that member.
1094 struct InlineRTPBranch {
1095 int instanceId;
1096 GstElement* queue = nullptr; // branch entry point (added to caller's bin)
1097 GstPad* teeSrcPad = nullptr; // requested pad on tee (must be released)
1098 };
1099 void FlushSendPipelines();
1100
1101 // Apply the master volume to every send pipeline. A sender's PipeWire
1102 // node is a capture stream, not a sink, so the master's pactl pass over
1103 // the output sinks never reaches it; the gain is a volume element in the
1104 // pipeline instead. New pipelines take it from getVolume() as they start.
1105 void SetMasterVolume(int volume);
1106
1107 AES67Manager();
1108 virtual ~AES67Manager();
1109
1110 private:
1111
1112 // Config
1113 AES67Config m_config;
1114 std::string m_configPath;
1115 bool LoadConfig();
1116
1117 // Guards every access to m_config.
1118 //
1119 // LoadConfig() clears and refills m_config.instances, which reallocates
1120 // the vector and reassigns its std::strings. ApplyConfig() and the SAP
1121 // threads are safe (ApplyConfig joins those threads before loading), but
1122 // GetStatus(), RunSelfTest() and the PTP query helpers all run on HTTP or
1123 // command threads and read m_config concurrently -- iterating a vector
1124 // that is being reallocated, or reading a std::string that is being
1125 // reassigned, is a use-after-free. UIs poll status, so this is the
1126 // likeliest crash in the field.
1127 //
1128 // Held only for the duration of a read or the final swap in LoadConfig(),
1129 // and NEVER together with m_pipelineMutex -- readers snapshot what they
1130 // need, release, then take the pipeline lock. That keeps the two locks
1131 // unordered with respect to each other and makes deadlock impossible.
1132 std::mutex m_configMutex;
1133
1134 // Snapshot accessors -- copy under the lock, use the copy afterwards.
1135 AES67Config GetConfigSnapshot();
1136 bool IsPtpEnabled();
1137 int GetPtpDomain();
1138 std::string GetPtpInterface();
1139
1140 // PTP (managed via ptp4l subprocess)
1141 bool m_ptpInitialized = false;
1142 pid_t m_ptp4lPid = -1; // ptp4l child process ID
1143 pid_t m_phc2sysPid = -1; // phc2sys child process ID
1144 // pidfds for the two, or -1 (no pidfd support). fppd's SA_NOCLDWAIT
1145 // reaps a child the moment it exits, so its PID can be reused at once;
1146 // a pidfd always refers to our child, where kill(pid) could hit
1147 // whatever took the number.
1148 int m_ptp4lPidfd = -1;
1149 // m_ptp4lPid and m_ptp4lPidfd: IsPtp4lRunning() reads them from HTTP
1150 // threads (status, self-test) while the main loop starts and stops
1151 // ptp4l, and closing the pidfd under a reader polled a closed -- or
1152 // reused -- descriptor. Held only to read or swap them, never across a
1153 // wait for the child.
1154 mutable std::mutex m_ptp4lProcMutex;
1155 // ptp4l is running with hardware timestamping, i.e. disciplining the
1156 // NIC's PHC. With software timestamping it disciplines CLOCK_REALTIME
1157 // and leaves the PHC free-running, so only this makes the PHC the
1158 // domain's time.
1159 std::atomic<bool> m_ptpHwTimestamping{false};
1160 int m_phc2sysPidfd = -1;
1161 void ForgetPtp4l();
1162 std::string m_ptpConfPath; // temp config file for ptp4l
1163 bool InitPTP();
1164 // Wait (up to ptpLockWaitMs) for the clock to settle before the streams
1165 // anchor to it, and record whether it did. Part of InitPTP(), and run on
1166 // its own when a rebuild keeps the running ptp4l.
1167 void AnchorToPtpLock();
1168 // What the running ptp4l was started with. A rebuild that leaves these
1169 // alone keeps it: restarting it drops every receiver in the domain off
1170 // the clock, and with this box as grandmaster that is all of them.
1171 struct PtpParams {
1172 std::string interface;
1173 int domain = 0;
1174 std::string role;
1175 11 bool operator==(const PtpParams& o) const {
1176
4/6
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✓ Branch 3 taken 4 times.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
11 return interface == o.interface && domain == o.domain && role == o.role;
1177 }
1178 };
1179 PtpParams m_ptpStartedWith;
1180 void ShutdownPTP();
1181 bool IsPtp4lRunning() const; // check if ptp4l process is alive
1182 std::string GetPTPClockId(); // EUI-64 from interface MAC (this node's own identity)
1183 std::string GetPtp4lState(); // query ptp4l port state via pmc (MASTER/SLAVE/LISTENING/...)
1184
1185 // Write an AES67-profile ptp4l config. Split out so the three start
1186 // attempts (hardware, software, software-without-DSCP) cannot drift apart
1187 // -- they used to be copy-pasted blocks.
1188 bool WritePtpConf(const std::string& path, bool hwTimestamping, bool includeDscp);
1189
1190 // fork/exec ptp4l with the given config. Returns true if it is still
1191 // alive shortly afterwards.
1192 bool StartPtp4l(bool hwTimestamping, bool includeDscp);
1193
1194 // What one poll of ptp4l says about whether the clock will still move
1195 // under anything anchored to it. See the comment above PtpSettledNow()
1196 // in the .cpp for what each outcome means and why the offset, rather than
1197 // the port state, is the signal.
1198 enum class PtpSettleState {
1199 NotSettled, // still converging, or the query told us nothing
1200 Grandmaster, // we hold the role -- the PHC is the domain's time
1201 SlaveLocked, // measured against the GM, correction already applied
1202 NoGrandmasterYet, // LISTENING and gmPresent actually read false
1203 };
1204
1205 // One poll of the above. Factored out of WaitForPtpLock() so the watchdog
1206 // can ask the same question later without a second copy of the answer to
1207 // drift out of step with this one. portState and offsetNs, when given,
1208 // receive the readings the verdict was reached on, for logging.
1209 //
1210 // NoGrandmasterYet is deliberately not folded into "settled" here: while
1211 // the initial wait is still running, "no announce seen" and "no announce
1212 // yet" look identical, so only a caller that knows it has waited long
1213 // enough may treat it as an answer.
1214 PtpSettleState PtpSettledNow(std::string* portState = nullptr,
1215 int64_t* offsetNs = nullptr);
1216
1217 // Block until ptp4l reports a clock that will not step under us, or until
1218 // timeoutMs elapses. Returns true if it settled. See
1219 // AES67::PTP_LOCK_OFFSET_NS for what "settled" means and why waiting for
1220 // it is a precondition of anchoring anything to PTP.
1221 bool WaitForPtpLock(int timeoutMs);
1222
1223 // Restart ptp4l/phc2sys if they have died (link flap, OOM, manual kill),
1224 // and notice a PTP clock that has stepped out from under the pipelines'
1225 // RTP anchors. Returns true when the caller should rebuild everything.
1226 // Called from WatchdogLoop() alongside the pipeline watchdog.
1227 bool CheckPtpWatchdog();
1228
1229 // Retract WARNING_PTP_NO_LOCK once the clock it complains about has
1230 // settled -- or ask for a rebuild, if settling moved the clock far enough
1231 // that the anchors really are wrong. Called from CheckPtpWatchdog();
1232 // returns true when the caller should rebuild.
1233 bool ReviewPtpNoLockWarning();
1234
1235 // Raise/retract WARNING_PTP_NO_LOCK. Nothing else should touch that
1236 // warning directly: the banner and the m_ptpNoLockWarned flag have to move
1237 // together, or the watchdog either never reviews a warning that is up or
1238 // reviews one that is not. (The baseline is seeded separately, by
1239 // ResetPtpStepReference() -- see there.)
1240 void RaisePtpNoLockWarning();
1241 void ClearPtpNoLockWarning();
1242
1243 // Step detector state: (PTP - CLOCK_MONOTONIC) as it stood at the last
1244 // sample. ResetPtpStepReference() takes one when ApplyConfig() has
1245 // finished building pipelines, so the first watchdog check 30s later can
1246 // already catch a step; PtpClockStepped() compares and re-samples, so the
1247 // comparison is only ever across one watchdog interval and steady drift
1248 // between the two crystals never accumulates into a false positive.
1249 // See AES67::PTP_STEP_DETECT_NS.
1250 int64_t m_ptpMonotonicDeltaNs = 0;
1251 bool m_ptpMonotonicDeltaValid = false;
1252 std::mutex m_ptpStepMutex;
1253 void ResetPtpStepReference();
1254 bool PtpClockStepped();
1255 // PTP minus CLOCK_MONOTONIC, or false when there is no PTP clock to read.
1256 bool PtpMonotonicDelta(int64_t& deltaNs);
1257
1258 // WARNING_PTP_NO_LOCK bookkeeping. m_ptpNoLockWarned says the banner is
1259 // up; the baseline is (PTP - CLOCK_MONOTONIC) as it stood when it went up,
1260 // which is the clock the pipelines were about to anchor to.
1261 //
1262 // The step detector's own reference cannot answer this question: it
1263 // re-samples on every pass, so it only ever sees one watchdog interval's
1264 // worth of movement, and a clock that slews half a second into lock over
1265 // ten minutes moves ~28ms per interval and never trips it. This baseline
1266 // is the one that does not move, so total distance travelled since the
1267 // anchors were built can be measured against it.
1268 //
1269 // Guarded by m_ptpStepMutex, except the atomic flag.
1270 std::atomic<bool> m_ptpNoLockWarned{false};
1271 int64_t m_ptpNoLockBaselineNs = 0;
1272 bool m_ptpNoLockBaselineValid = false;
1273
1274 // Whether the current pipelines' RTP anchors can be trusted: false while
1275 // they are known to have been built against a clock that had not settled.
1276 // Set by InitPTP() from the WaitForPtpLock() result, and cleared again by
1277 // ReviewPtpNoLockWarning() if the clock settles close enough to where the
1278 // anchors were placed that they turned out fine after all. Reported as
1279 // ptp.lockedAtStart, which drives the "streams started before PTP settled"
1280 // badge -- so it has to stop being false whenever the warning does, or the
1281 // page keeps promising a rebuild that is no longer coming.
1282 std::atomic<bool> m_ptpLockedAtStart{false};
1283
1284 // Query the actual PTP grandmaster (may be a remote clock, not this node)
1285 // via `pmc GET TIME_STATUS_NP`. Returns false if ptp4l isn't running or
1286 // the query fails, leaving the out-params untouched.
1287 //
1288 // offsetNs is only meaningful when offsetValid says so: a missing or
1289 // malformed master_offset leaves it at zero, which is indistinguishable
1290 // from a perfectly locked clock. Ask for it anywhere the number is shown
1291 // or acted on; the grandmaster identity alone does not need it.
1292 bool QueryPtp4lTimeStatus(bool& gmPresent, std::string& gmIdentity, int64_t& offsetNs,
1293 bool* offsetValid = nullptr);
1294
1295 // The AES67 media clock -- PTP time, shared by every send pipeline so they
1296 // all sit on one timeline. Created on first use, released in Shutdown().
1297 // Null when PTP is disabled or no PTP time source could be opened, in
1298 // which case pipelines fall back to GStreamer's default clock.
1299 GstClock* m_ptpClock = nullptr;
1300 std::mutex m_ptpClockMutex;
1301 GstClock* GetOrCreateMediaClock();
1302 void ReleaseMediaClock();
1303
1304 // Grandmaster identity to advertise in SDP / report over HTTP: the remote
1305 // clock we follow, or our own identity when we hold the role. Empty when
1306 // PTP is enabled but no grandmaster has been settled on yet.
1307 std::string GetActiveGrandmasterId();
1308
1309 // Cached pmc results -- see AES67::PTP_QUERY_CACHE_MS.
1310 struct PtpQueryCache {
1311 std::chrono::steady_clock::time_point when{};
1312 bool valid = false;
1313 bool gmPresent = false;
1314 bool gmPresentValid = false;
1315 std::string gmIdentity;
1316 int64_t offsetNs = 0;
1317 bool offsetValid = false;
1318 std::string portState;
1319 };
1320 PtpQueryCache m_ptpCache;
1321 std::mutex m_ptpCacheMutex;
1322 void RefreshPtpCache(bool force = false);
1323
1324 // Where a grandmaster identity is announcing from. Populated by the
1325 // Announce listener below, which is the only source of this: pmc reports
1326 // clock identities, never addresses, and a clock identity is an EUI-64
1327 // derived from a MAC that need not belong to the interface carrying PTP,
1328 // so it cannot be resolved to an address by inspection.
1329 struct PtpAnnounceSource {
1330 std::string address;
1331 bool direct = false; // sender IS the grandmaster (stepsRemoved 0)
1332 int64_t lastSeenMs = 0;
1333 };
1334 std::map<std::string, PtpAnnounceSource> m_ptpAnnounceSources; // keyed by GM id
1335 std::mutex m_ptpAnnounceMutex;
1336 std::thread m_ptpAnnounceThread;
1337 std::atomic<bool> m_ptpAnnounceRunning{false};
1338 void PtpAnnounceListenLoop();
1339 void HandlePtpAnnounce(const uint8_t* data, size_t len, const std::string& senderAddr);
1340
1341 // Address the given grandmaster is currently announcing from. False when
1342 // nothing recent has been heard from it, leaving the out-params untouched.
1343 bool GetGrandmasterAddress(const std::string& gmId, std::string& address, bool& viaBoundary);
1344
1345 // Sink-pacing timestamp shift per instance, updatable once the pipeline
1346 // reports its real latency. Owned here so the pad probe can read a live
1347 // value rather than one baked in before the latency is known.
1348 std::map<int, std::atomic<GstClockTime>*> m_sinkPacingShift;
1349
1350 // Drift control loop -- see AES67Config::adaptiveResample. Runs on its own
1351 // thread because it has to sample far more often than the 30s watchdog.
1352 std::thread m_driftThread;
1353 std::atomic<bool> m_driftRunning{false};
1354 void DriftControlLoop();
1355
1356 // Pipeline watchdog — called from WatchdogLoop() to poll bus messages
1357 // and restart any pipeline that isn't in PLAYING state.
1358 // Returns true if a full pipeline rebuild is needed.
1359 bool PollPipelinesWatchdog();
1360 // Rebuilds asked for by bus errors, for the backoff. A rebuild restarts
1361 // ptp4l for the whole domain, so an error that comes straight back must
1362 // not do that every pass. Watchdog thread only.
1363 int m_errorRebuilds = 0;
1364 // Streams whose pipeline failed to build or start in the last apply
1365 // (PipeWire not up yet, say). Nothing else would try them again, so the
1366 // watchdog rebuilds for them on the bus-error backoff. Set by
1367 // ApplyConfig() under m_pipelineMutex.
1368 int m_failedStarts = 0;
1369 // PTP enabled but not running (its start, or a watchdog restart, failed):
1370 // when the watchdog tries again, backing off 30s..5min. Watchdog thread
1371 // only; reset by a successful start.
1372 std::chrono::steady_clock::time_point m_ptpRetryAt{};
1373 int m_ptpRetryDelayS = 0;
1374 std::chrono::steady_clock::time_point m_lastErrorRebuild{};
1375
1376 // Deferred pipeline-rebuild thread — spawned by WatchdogLoop()'s
1377 // watchdog rebuild path to call ApplyConfig() from a thread other than
1378 // the one ApplyConfig() needs to join (m_sapAnnounceThread). Tracked
1379 // as a member (rather than detached) so Shutdown() can join it before
1380 // tearing anything else down -- see Shutdown() and WatchdogLoop().
1381 std::thread m_rebuildThread;
1382
1383 // Rebuild bookkeeping. m_rebuildMutex guards m_rebuildThread. A rebuild
1384 // is scheduled with the apply generation it was asked for and runs only
1385 // if nothing has applied, cleaned up or shut down since (see
1386 // ScheduleRebuild()); m_stopping is set once by Shutdown() and keeps
1387 // ApplyConfig() from bringing anything back after it.
1388 std::mutex m_rebuildMutex;
1389 std::atomic<bool> m_rebuildRunning{false};
1390 std::atomic<bool> m_stopping{false};
1391 std::atomic<uint64_t> m_applyGeneration{0};
1392 void ScheduleRebuild();
1393 bool ApplyConfigLocked();
1394
1395 // Serializes the whole tear-down/rebuild sequence in ApplyConfig(),
1396 // Shutdown() and Cleanup(). ApplyConfig() joins the SAP threads at the
1397 // top and re-creates them ~70 lines later, after loading config and
1398 // building pipelines; that window is wide, and ApplyConfig() has three
1399 // independent callers (the boot sequence, AES67ApplyCommand::run on a
1400 // command thread, and m_rebuildThread from the watchdog). Two overlapping
1401 // calls both pass the join, then both assign m_sapAnnounceThread -- and
1402 // assigning over a still-joinable std::thread is an unconditional
1403 // std::terminate().
1404 //
1405 // Shutdown() must join m_rebuildThread BEFORE taking this lock: that
1406 // thread calls ApplyConfig(), so holding the lock across the join would
1407 // deadlock.
1408 std::mutex m_applyMutex;
1409
1410 // Pipeline management
1411 std::map<int, AES67Pipeline> m_sendPipelines; // keyed by instance ID
1412 std::map<int, AES67Pipeline> m_recvPipelines;
1413 std::mutex m_pipelineMutex;
1414
1415 // Send instances ApplyConfig() chose not to start because nothing feeds
1416 // them -- see AES67Config::requireGroupSource. Instance ID to the reason,
1417 // which is shown to the user, so phrase it as guidance rather than as an
1418 // error. Guarded by m_pipelineMutex alongside the pipeline maps, so a
1419 // status reader sees the whole picture -- running, failed and waiting --
1420 // under one lock.
1421 std::map<int, std::string> m_deferredSenders;
1422
1423 bool CreateSendPipeline(const AES67Instance& inst);
1424 bool CreateRecvPipeline(const AES67Instance& inst);
1425 void StopPipeline(AES67Pipeline& p);
1426 void StopAllPipelines();
1427
1428 // Get source IP for the network interface
1429 std::string GetInterfaceIP(const std::string& iface);
1430
1431 // SAP announcer thread
1432 std::thread m_sapAnnounceThread;
1433 std::atomic<bool> m_sapAnnounceRunning{false};
1434 void SAPAnnounceLoop();
1435
1436 // Recovery and PTP supervision; see WatchdogLoop().
1437 std::thread m_watchdogThread;
1438 std::atomic<bool> m_watchdogRunning{false};
1439 void WatchdogLoop();
1440 void StopWorkerThreads();
1441 std::string BuildSDP(const AES67Instance& inst, const std::string& sourceIP,
1442 const std::string& ptpClockId, uint32_t sdpVersion);
1443 std::vector<uint8_t> BuildSAPPacket(const std::string& sourceIP,
1444 uint16_t msgIdHash,
1445 const std::string& sdp,
1446 bool isDeletion = false);
1447
1448 // The SDP for one instance as the announcer would emit it right now, for
1449 // the /aes67/sdp export. Deliberately the same BuildSDP() the SAP path
1450 // uses: a file the user hands to Stream Monitor or VLC has to describe the
1451 // stream that is actually on the wire, refclk and all, or it is worse than
1452 // no file at all when the point of it is troubleshooting.
1453 std::string ExportSDPFor(const AES67Instance& inst, const AES67Config& cfg);
1454
1455 // RFC 2974 requires the message id hash to change whenever the announced
1456 // payload changes, so it is computed over the SDP text itself rather than
1457 // over the instance -- otherwise receivers dedupe an updated announcement
1458 // as a repeat and never see the corrected refclk.
1459 uint16_t ComputeSAPHash(const std::string& sdp);
1460
1461 // SDP o= version for a given announcement body. Only advances when the
1462 // announced content actually changes: the msg id hash is derived from the
1463 // SDP text, so bumping the version on every rebuild would mint a new hash
1464 // each time and leave SAP receivers listing a phantom stream per restart
1465 // until their timeout (~10 announce intervals) cleared it.
1466 //
1467 // The mapping is persisted so an fppd restart with unchanged config
1468 // re-announces byte-identical SDP rather than a new session.
1469 uint32_t SDPVersionFor(const std::string& body);
1470 void LoadSDPVersion();
1471 void SaveSDPVersion();
1472 std::string m_sdpVersionPath;
1473 std::string m_sdpBodyKey; // hash of the last announced body
1474 uint32_t NextSDPVersion();
1475 // Bumped from the SAP thread on every rebuild and from RunSelfTest() on
1476 // whichever thread asked for the test, so it cannot be a plain int.
1477 std::atomic<uint32_t> m_lastSdpVersion{0};
1478
1479 // SAP receiver thread
1480 std::thread m_sapRecvThread;
1481 std::atomic<bool> m_sapRecvRunning{false};
1482 void SAPReceiveLoop();
1483 // Keyed by (origin address, message id hash): RFC 2974 identifies an
1484 // announcement by both, and the 16-bit hash alone collides between
1485 // senders.
1486 std::map<std::pair<std::string, uint16_t>, SAPDiscoveredStream> m_discoveredStreams;
1487 std::mutex m_discoveredMutex;
1488 void HandleSAPPacket(const uint8_t* data, size_t len,
1489 const std::string& senderAddr);
1490
1491 // State
1492 std::atomic<bool> m_active{false};
1493 std::atomic<bool> m_initialized{false};
1494
1495 // Safe node name (matches fpp_aes67_common.py safe_node_name)
1496 static std::string SafeNodeName(const std::string& name);
1497 };
1498
1499 #endif // HAS_AES67_GSTREAMER
1500