1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <stdint.h>
#include <atomic>
#include <deque>
#include <memory>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
#include "absl/types/optional.h"
#include "api/frame_transformer_interface.h"
#include "api/rtp_sender_interface.h"
#include "api/rtp_receiver_interface.h"
#include "api/scoped_refptr.h"
#include "livekit/webrtc.h"
#include "rtc_base/synchronization/mutex.h"
#include "rust/cxx.h"
// Forward declarations to avoid circular includes
// (video_track.h -> packet_trailer.h -> peer_connection.h -> media_stream.h -> video_track.h)
namespace livekit_ffi {
class PeerConnectionFactory;
class RtpSender;
class RtpReceiver;
} // namespace livekit_ffi
namespace livekit_ffi {
// Magic bytes to identify packet trailers: "LKTS" (LiveKit TimeStamp)
constexpr uint8_t kPacketTrailerMagic[4] = {'L', 'K', 'T', 'S'};
// Trailer envelope: [trailer_len: 1B] [magic: 4B] = 5 bytes.
// Always present at the end of every trailer.
constexpr size_t kTrailerEnvelopeSize = 5;
// TLV element overhead: [tag: 1B] [len: 1B] = 2 bytes before value.
// All TLV bytes (tag, len, value) are XORed with 0xFF.
// TLV tag IDs
constexpr uint8_t kTagTimestampUs = 0x01; // value: 8 bytes big-endian uint64
constexpr uint8_t kTagFrameId = 0x02; // value: 4 bytes big-endian uint32
constexpr size_t kTimestampTlvSize = 10; // tag + len + 8-byte value
constexpr size_t kFrameIdTlvSize = 6; // tag + len + 4-byte value
// Trailer size varies because frame_id is omitted when it is unset (0).
constexpr size_t kPacketTrailerMinSize =
kTimestampTlvSize + kTrailerEnvelopeSize;
constexpr size_t kPacketTrailerMaxSize =
kTimestampTlvSize + kFrameIdTlvSize + kTrailerEnvelopeSize;
struct PacketTrailerMetadata {
uint64_t user_timestamp;
uint32_t frame_id;
uint32_t ssrc; // SSRC that produced this entry (for simulcast tracking)
};
/// Frame transformer that appends/extracts packet trailers.
/// This transformer can be used standalone or in conjunction with e2ee.
///
/// On the send side, user timestamps are stored in an internal map keyed
/// by capture timestamp (microseconds). When TransformSend fires it
/// looks up the user timestamp via the frame's CaptureTime().
///
/// On the receive side, extracted frame metadata is stored in an
/// internal map keyed by RTP timestamp (uint32_t). Decoded frames can
/// look up their metadata via lookup_frame_metadata(rtp_ts).
class PacketTrailerTransformer : public webrtc::FrameTransformerInterface {
public:
enum class Direction { kSend, kReceive };
explicit PacketTrailerTransformer(Direction direction);
~PacketTrailerTransformer() override = default;
// FrameTransformerInterface implementation
void Transform(
std::unique_ptr<webrtc::TransformableFrameInterface> frame) override;
void RegisterTransformedFrameCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback> callback) override;
void RegisterTransformedFrameSinkCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback> callback,
uint32_t ssrc) override;
void UnregisterTransformedFrameCallback() override;
void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) override;
/// Enable/disable timestamp embedding
void set_enabled(bool enabled);
bool enabled() const;
/// Lookup the frame metadata associated with a given RTP timestamp.
/// Returns the metadata if found, nullopt otherwise.
/// The entry is removed from the map after lookup.
std::optional<PacketTrailerMetadata> lookup_frame_metadata(uint32_t rtp_timestamp);
/// Store frame metadata for a given capture timestamp (sender side).
/// Called from VideoTrackSource::on_captured_frame with the
/// TimestampAligner-adjusted timestamp, which matches CaptureTime()
/// in the encoder pipeline.
void store_frame_metadata(int64_t capture_timestamp_us,
uint64_t user_timestamp,
uint32_t frame_id);
private:
void TransformSend(
std::unique_ptr<webrtc::TransformableFrameInterface> frame);
void TransformReceive(
std::unique_ptr<webrtc::TransformableFrameInterface> frame);
/// Append frame metadata trailer to frame data
std::vector<uint8_t> AppendTrailer(
webrtc::ArrayView<const uint8_t> data,
uint64_t user_timestamp,
uint32_t frame_id);
/// Extract and remove frame metadata trailer from frame data
std::optional<PacketTrailerMetadata> ExtractTrailer(
webrtc::ArrayView<const uint8_t> data,
std::vector<uint8_t>& out_data);
const Direction direction_;
std::atomic<bool> enabled_{true};
mutable webrtc::Mutex mutex_;
webrtc::scoped_refptr<webrtc::TransformedFrameCallback> callback_;
std::unordered_map<uint32_t,
webrtc::scoped_refptr<webrtc::TransformedFrameCallback>>
sink_callbacks_;
// Send-side map: capture timestamp (us) -> frame metadata.
// Populated by store_frame_metadata(), consumed by TransformSend()
// via CaptureTime() lookup.
mutable webrtc::Mutex send_map_mutex_;
mutable std::unordered_map<int64_t, PacketTrailerMetadata> send_map_;
mutable std::deque<int64_t> send_map_order_;
static constexpr size_t kMaxSendMapEntries = 300;
// Receive-side map: RTP timestamp -> frame metadata.
// Keyed by RTP timestamp so decoded frames can look up their
// metadata regardless of frame drops or reordering.
mutable webrtc::Mutex recv_map_mutex_;
mutable std::unordered_map<uint32_t, PacketTrailerMetadata> recv_map_;
mutable std::deque<uint32_t> recv_map_order_;
static constexpr size_t kMaxRecvMapEntries = 300;
// Simulcast tracking: detect layer switches and flush stale entries.
mutable uint32_t recv_active_ssrc_{0};
};
/// Wrapper class for Rust FFI that manages packet trailer transformers.
class PacketTrailerHandler {
public:
PacketTrailerHandler(
std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::RtpSenderInterface> sender);
PacketTrailerHandler(
std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver);
~PacketTrailerHandler() = default;
/// Enable/disable timestamp embedding
void set_enabled(bool enabled) const;
bool enabled() const;
/// Lookup the user timestamp for a given RTP timestamp (receiver side).
/// Returns UINT64_MAX if not found. The entry is removed after lookup.
/// Also caches the frame_id for retrieval via last_lookup_frame_id().
uint64_t lookup_timestamp(uint32_t rtp_timestamp) const;
/// Returns the frame_id from the most recent successful
/// lookup_timestamp() call. Returns 0 if no lookup succeeded.
uint32_t last_lookup_frame_id() const;
/// Store frame metadata for a given capture timestamp (sender side).
void store_frame_metadata(int64_t capture_timestamp_us,
uint64_t user_timestamp,
uint32_t frame_id) const;
/// Access the underlying transformer for chaining.
webrtc::scoped_refptr<PacketTrailerTransformer> transformer() const;
private:
std::shared_ptr<RtcRuntime> rtc_runtime_;
webrtc::scoped_refptr<PacketTrailerTransformer> transformer_;
webrtc::scoped_refptr<webrtc::RtpSenderInterface> sender_;
webrtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver_;
mutable uint32_t last_frame_id_{0};
};
// Factory functions for Rust FFI
std::shared_ptr<PacketTrailerHandler> new_packet_trailer_sender(
std::shared_ptr<PeerConnectionFactory> peer_factory,
std::shared_ptr<RtpSender> sender);
std::shared_ptr<PacketTrailerHandler> new_packet_trailer_receiver(
std::shared_ptr<PeerConnectionFactory> peer_factory,
std::shared_ptr<RtpReceiver> receiver);
} // namespace livekit_ffi