clock_bound/shm.rs
1//! ClockBound Shared Memory
2//!
3//! This crate implements the low-level IPC functionality to share `ClockErrorBound` data and clock
4//! status over a shared memory segment. This crate is meant to be used by the C and Rust versions
5//! of the ClockBound client library.
6
7// TODO: prevent clippy from checking for dead code. The writer module is only re-exported publicly
8// if the write feature is selected. There may be a better way to do that and re-enable the lint.
9#![expect(dead_code)]
10
11pub mod common;
12mod reader;
13mod shm_header;
14mod tsc;
15mod writer;
16
17// Re-exports reader and writer. The writer is conditionally included under the "writer" feature.
18use common::{CLOCK_MONOTONIC, CLOCK_REALTIME, clock_gettime_safe};
19pub use reader::ShmReader;
20use tsc::read_timestamp_counter_begin;
21pub use writer::{ShmWrite, ShmWriter};
22
23use bon::Builder;
24use errno::Errno;
25use nix::sys::time::{TimeSpec, TimeValLike};
26use std::error::Error;
27use std::fmt;
28
29pub const CLOCKBOUND_SHM_DEFAULT_PATH_V0: &str = "/var/run/clockbound/shm0";
30pub const CLOCKBOUND_SHM_DEFAULT_PATH_V1: &str = "/var/run/clockbound/shm1";
31pub const CLOCKBOUND_SHM_CLIENT_DEFAULT_PATH: &str = CLOCKBOUND_SHM_DEFAULT_PATH_V1;
32
33const FREE_RUNNING_GRACE_PERIOD: TimeSpec = TimeSpec::new(60, 0);
34const NANOS_PER_SECOND: f64 = 1_000_000_000.0;
35
36/// Convenience macro to build a `ShmError::SyscallError` with extra info from errno and custom
37/// origin information.
38#[macro_export]
39macro_rules! syserror {
40 ($msg:expr) => {
41 Err($crate::shm::ShmError::SyscallError($msg, ::errno::errno()))
42 };
43}
44
45pub trait ClockBoundSnapshot {
46 /// The `ClockErrorBound` equivalent of `clock_gettime()`, but with bound on accuracy.
47 ///
48 /// Returns a `ClockBoundNowResult` with contains the (earliest, latest) timespec between which
49 /// current time exists. The interval width is twice the clock error bound (ceb) such that:
50 /// (earliest, latest) = ((now - ceb), (now + ceb))
51 ///
52 /// The function also returns a clock status to assert that the clock is being synchronized, or
53 /// free-running, or ...
54 #[expect(clippy::missing_errors_doc, reason = "todo")]
55 fn now(&self) -> Result<ClockBoundNowResult, ShmError>;
56}
57
58/// Enum that holds supported layout of the `ClockErrorBound` stored in the ClockBound daemon
59/// shared memory segment.
60#[derive(Debug, Copy, Clone, PartialEq)]
61pub enum ClockErrorBound {
62 V2(ClockErrorBoundV2),
63 V3(ClockErrorBoundV3),
64}
65
66impl ClockErrorBound {
67 pub fn as_of(&self) -> TimeSpec {
68 match self {
69 ClockErrorBound::V2(ceb) => ceb.as_of,
70 ClockErrorBound::V3(ceb) => ceb.as_of,
71 }
72 }
73
74 pub fn void_after(&self) -> TimeSpec {
75 match self {
76 ClockErrorBound::V2(ceb) => ceb.void_after,
77 ClockErrorBound::V3(ceb) => ceb.void_after,
78 }
79 }
80
81 pub fn bound_nsec(&self) -> i64 {
82 match self {
83 ClockErrorBound::V2(ceb) => ceb.bound_nsec,
84 ClockErrorBound::V3(ceb) => ceb.bound_nsec,
85 }
86 }
87
88 pub fn max_drift_ppb(&self) -> u32 {
89 match self {
90 ClockErrorBound::V2(ceb) => ceb.max_drift_ppb,
91 ClockErrorBound::V3(ceb) => ceb.max_drift_ppb,
92 }
93 }
94
95 pub fn clock_status(&self) -> ClockStatus {
96 match self {
97 ClockErrorBound::V2(ceb) => ceb.clock_status,
98 ClockErrorBound::V3(ceb) => ceb.clock_status,
99 }
100 }
101
102 pub fn disruption_marker(&self) -> u64 {
103 match self {
104 ClockErrorBound::V2(ceb) => ceb.disruption_marker,
105 ClockErrorBound::V3(ceb) => ceb.disruption_marker,
106 }
107 }
108
109 pub fn clock_disruption_support_enabled(&self) -> bool {
110 match self {
111 ClockErrorBound::V2(ceb) => ceb.clock_disruption_support_enabled,
112 ClockErrorBound::V3(ceb) => ceb.clock_disruption_support_enabled,
113 }
114 }
115}
116
117impl ClockBoundSnapshot for ClockErrorBound {
118 fn now(&self) -> Result<ClockBoundNowResult, ShmError> {
119 match self {
120 ClockErrorBound::V2(ceb) => ceb.now(),
121 ClockErrorBound::V3(ceb) => ceb.now(),
122 }
123 }
124}
125
126/// Generic `ClockErrorBound` builder.
127#[derive(Builder)]
128// Rename auto-generated build() function into build_internal so we have a custom finishing
129// function to create the enum variants
130#[builder(finish_fn(vis = "", name = build_internal))]
131pub struct ClockErrorBoundGeneric {
132 #[builder(default)]
133 as_of_tsc: u64,
134
135 #[builder(default = TimeSpec::new(0, 0))]
136 as_of: TimeSpec,
137
138 #[builder(default = TimeSpec::new(0, 0))]
139 void_after: TimeSpec,
140
141 #[builder(default)]
142 bound_nsec: i64,
143
144 /// V3 only: the period in seconds per tick.
145 #[builder(default)]
146 period: f64,
147
148 /// V3 only: the relative error of the period, a dimensionless ratio in `[0, 1)`.
149 #[builder(default)]
150 period_err: f64,
151
152 #[builder(default)]
153 disruption_marker: u64,
154
155 #[builder(default)]
156 max_drift_ppb: u32,
157
158 #[builder(default = ClockStatus::Unknown)]
159 clock_status: ClockStatus,
160
161 #[builder(default)]
162 clock_disruption_support_enabled: bool,
163}
164
165impl<S: clock_error_bound_generic_builder::IsComplete> ClockErrorBoundGenericBuilder<S> {
166 /// Custom `build` finishing function on the generated `ClockErrorBoundLayoutBuilder`.
167 ///
168 /// Take the layout version number as a parameter, it is a u16 to ease casting of the earlier
169 /// version of the `SHMHeader`.
170 pub fn build(self, layout_version: ClockErrorBoundLayoutVersion) -> ClockErrorBound {
171 // Build the ClockErrorBoundGeneric object
172 let ceb = self.build_internal();
173
174 // Build the specific version of the ClockErrorBound
175 match layout_version {
176 ClockErrorBoundLayoutVersion::V2 => ClockErrorBound::V2(ClockErrorBoundV2::new(
177 ceb.as_of,
178 ceb.void_after,
179 ceb.bound_nsec,
180 ceb.disruption_marker,
181 ceb.max_drift_ppb,
182 ceb.clock_status,
183 ceb.clock_disruption_support_enabled,
184 )),
185 ClockErrorBoundLayoutVersion::V3 => ClockErrorBound::V3(ClockErrorBoundV3::new(
186 ceb.as_of_tsc,
187 ceb.as_of,
188 ceb.void_after,
189 ceb.period,
190 ceb.period_err,
191 ceb.bound_nsec,
192 ceb.disruption_marker,
193 ceb.max_drift_ppb,
194 ceb.clock_status,
195 ceb.clock_disruption_support_enabled,
196 )),
197 }
198 }
199}
200
201#[derive(Copy, Clone)]
202pub enum ClockErrorBoundLayoutVersion {
203 V2,
204 V3,
205}
206
207impl TryFrom<u8> for ClockErrorBoundLayoutVersion {
208 type Error = ShmError;
209 fn try_from(value: u8) -> Result<Self, ShmError> {
210 match value {
211 2 => Ok(ClockErrorBoundLayoutVersion::V2),
212 3 => Ok(ClockErrorBoundLayoutVersion::V3),
213 _ => Err(ShmError::SegmentVersionNotSupported(format!(
214 "Found version {value}",
215 ))),
216 }
217 }
218}
219
220impl TryFrom<u16> for ClockErrorBoundLayoutVersion {
221 type Error = ShmError;
222 fn try_from(value: u16) -> Result<Self, ShmError> {
223 match value {
224 2 => Ok(ClockErrorBoundLayoutVersion::V2),
225 3 => Ok(ClockErrorBoundLayoutVersion::V3),
226 _ => Err(ShmError::SegmentVersionNotSupported(format!(
227 "Found version {value}",
228 ))),
229 }
230 }
231}
232
233impl From<ClockErrorBoundLayoutVersion> for u16 {
234 fn from(value: ClockErrorBoundLayoutVersion) -> Self {
235 match value {
236 ClockErrorBoundLayoutVersion::V2 => 2,
237 ClockErrorBoundLayoutVersion::V3 => 3,
238 }
239 }
240}
241
242/// Result of the `ClockBoundClient::now()` function.
243#[derive(PartialEq, Clone, Debug)]
244pub struct ClockBoundNowResult {
245 pub earliest: TimeSpec,
246 pub latest: TimeSpec,
247 pub clock_status: ClockStatus,
248}
249
250/// Error condition returned by all low-level ClockBound APIs.
251///
252#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
253pub enum ShmError {
254 /// A system call failed.
255 /// Variant includes the Errno struct with error details, and an indication on the origin of
256 /// the system call that error'ed.
257 SyscallError(String, Errno),
258
259 /// The shared memory segment is not initialized.
260 SegmentNotInitialized(String),
261
262 /// The shared memory segment is initialized but malformed.
263 SegmentMalformed(String),
264
265 /// Failed causality check when comparing timestamps.
266 CausalityBreach(String),
267
268 /// The shared memory segment version is not supported.
269 SegmentVersionNotSupported(String),
270}
271
272impl fmt::Display for ShmError {
273 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
274 match self {
275 ShmError::SyscallError(msg, errno) => {
276 write!(f, "Errno: {errno:?} Details: {msg}")
277 }
278 ShmError::SegmentNotInitialized(msg) => {
279 write!(f, "The shared memory segment is not initialized [{msg}].")
280 }
281 ShmError::SegmentMalformed(msg) => {
282 write!(
283 f,
284 "The shared memory segment is initialized but malformed [{msg}]."
285 )
286 }
287 ShmError::CausalityBreach(msg) => {
288 write!(
289 f,
290 "Failed causality check when comparing timestamps [{msg}]."
291 )
292 }
293 ShmError::SegmentVersionNotSupported(msg) => {
294 write!(
295 f,
296 "The shared memory segment version is not supported [{msg}]."
297 )
298 }
299 }
300 }
301}
302
303impl Error for ShmError {}
304
305/// Definition of mutually exclusive clock status exposed to the reader.
306///
307/// Note the data layout is explicitly set to i32. This enum is a field of the ClockBound shared
308/// memory segment, and its representation *may* be different for C code compiled with specific
309/// flags. Making it explicit removes this risk and ambiguity.
310#[repr(i32)]
311#[derive(Debug, Copy, Clone, PartialEq)]
312pub enum ClockStatus {
313 /// The status of the clock is unknown.
314 /// In this clock status, error-bounded timestamps should not be trusted.
315 Unknown = 0,
316
317 /// The clock is kept accurate by the synchronization daemon.
318 /// In this clock status, error-bounded timestamps can be trusted.
319 Synchronized = 1,
320
321 /// The clock is free running and not updated by the synchronization daemon.
322 /// In this clock status, error-bounded timestamps can be trusted.
323 FreeRunning = 2,
324
325 /// The clock has been disrupted and the accuracy of time cannot be bounded.
326 /// In this clock status, error-bounded timestamps should not be trusted.
327 Disrupted = 3,
328}
329
330/// Structure that holds the `ClockErrorBound` data captured at a specific point in time and valid
331/// until a subsequent point in time.
332///
333/// The `ClockErrorBound` structure supports calculating the actual bound on clock error at any time,
334/// using its `now()` method. The internal fields are not meant to be accessed directly.
335///
336/// Note that the timestamps in between which this `ClockErrorBound` data is valid are captured using
337/// a `CLOCK_MONOTONIC_COARSE` clock. The monotonic clock id is required to correctly measure the
338/// duration during which clock drift possibly accrues, and avoid events when the clock is set,
339/// smeared or affected by leap seconds.
340///
341/// The structure is shared across the Shared Memory segment and has a C representation to enforce
342/// this specific layout.
343#[repr(C)]
344#[derive(Debug, Copy, Clone, PartialEq)]
345pub struct ClockErrorBoundV2 {
346 /// The `CLOCK_MONOTONIC_COARSE` timestamp recorded when the bound on clock error was
347 /// calculated. The current implementation relies on Chrony tracking data, which accounts for
348 /// the dispersion between the last clock processing event, and the reading of tracking data.
349 as_of: TimeSpec,
350
351 /// The `CLOCK_MONOTONIC_COARSE` timestamp beyond which the bound on clock error should not be
352 /// trusted. This is a useful signal that the communication with the synchronization daemon is
353 /// has failed, for example.
354 void_after: TimeSpec,
355
356 /// An absolute upper bound on the accuracy of the `CLOCK_REALTIME` clock with regards to true
357 /// time at the instant represented by `as_of`.
358 bound_nsec: i64,
359
360 /// Disruption marker.
361 ///
362 /// This value is incremented (by an unspecified delta) each time the clock has been disrupted.
363 /// This count value is specific to a particular VM/EC2 instance.
364 disruption_marker: u64,
365
366 /// Maximum drift rate of the clock between updates of the synchronization daemon. The value
367 /// stored in `bound_nsec` should increase by the following to account for the clock drift
368 /// since `bound_nsec` was computed:
369 /// `bound_nsec += max_drift_ppb * (now - as_of)`
370 max_drift_ppb: u32,
371
372 /// The synchronization daemon status indicates whether the daemon is synchronized,
373 /// free-running, etc.
374 clock_status: ClockStatus,
375
376 /// Clock disruption support enabled flag.
377 ///
378 /// This indicates whether or not the ClockBound daemon was started with a
379 /// configuration that supports detecting clock disruptions.
380 clock_disruption_support_enabled: bool,
381
382 /// Padding.
383 _padding: [u8; 7],
384}
385
386impl ClockErrorBoundV2 {
387 /// Create a new `ClockErrorBound` struct.
388 pub fn new(
389 as_of: TimeSpec,
390 void_after: TimeSpec,
391 bound_nsec: i64,
392 disruption_marker: u64,
393 max_drift_ppb: u32,
394 clock_status: ClockStatus,
395 clock_disruption_support_enabled: bool,
396 ) -> ClockErrorBoundV2 {
397 ClockErrorBoundV2 {
398 as_of,
399 void_after,
400 bound_nsec,
401 disruption_marker,
402 max_drift_ppb,
403 clock_status,
404 clock_disruption_support_enabled,
405 _padding: [0u8; 7],
406 }
407 }
408
409 /// The `ClockErrorBoundV2` implementation of `now()`, a `clock_gettime()` equivalent but with
410 /// bound on clock accuracy.
411 ///
412 /// Returns a pair of (earliest, latest) timespec between which current time exists. The
413 /// interval width is twice the clock error bound (ceb) such that:
414 /// (earliest, latest) = ((now - ceb), (now + ceb))
415 /// The function also returns a clock status to assert that the clock is being synchronized, or
416 /// free-running, or ...
417 #[expect(clippy::missing_errors_doc, reason = "todo")]
418 pub fn now(&self) -> Result<ClockBoundNowResult, ShmError> {
419 // Read the clock, start with the REALTIME one to be as close as possible to the event the
420 // caller is interested in. The monotonic clock should be read after. It is correct for the
421 // process be preempted between the two calls: a delayed read of the monotonic clock will
422 // make the bound on clock error more pessimistic, but remains correct.
423 let real = clock_gettime_safe(CLOCK_REALTIME)?;
424 let mono = clock_gettime_safe(CLOCK_MONOTONIC)?;
425
426 self.compute_bound_at(real, mono)
427 }
428
429 /// Compute the bound on clock error at a given point in time.
430 ///
431 /// The time at which the bound is computed is defined by the (real, mono) pair of timestamps
432 /// read from the realtime and monotonic clock respectively, *roughly* at the same time. The
433 /// details to correctly work around the "rough" alignment of the timestamps is not something
434 /// we want to leave to the user of ClockBound, hence this method is private. Although `now()`
435 /// may be it only caller, decoupling the two make writing unit tests a bit easier.
436 #[expect(
437 clippy::cast_precision_loss,
438 clippy::cast_possible_truncation,
439 reason = "todo, come back and evaluate impact"
440 )]
441 fn compute_bound_at(
442 &self,
443 real: TimeSpec,
444 mono: TimeSpec,
445 ) -> Result<ClockBoundNowResult, ShmError> {
446 // Sanity checks:
447 // - `now()` should operate on a consistent snapshot of the shared memory segment, and
448 // causality between mono and as_of should be enforced.
449 // - a extremely high value of the `max_drift_ppb` is a sign of something going wrong
450 if self.max_drift_ppb >= 1_000_000_000 {
451 return Err(ShmError::SegmentMalformed(format!(
452 "max_drift_ppb too large [{}]",
453 self.max_drift_ppb,
454 )));
455 }
456
457 // If the ClockErrorBound data has not been updated "recently", the status of the clock
458 // cannot be guaranteed. Things are ambiguous, the synchronization daemon may be dead, or
459 // its interaction with the clockbound daemon is broken, or ... In any case, we signal the
460 // caller that guarantees are gone. We could return an Err here, but choosing to leverage
461 // ClockStatus instead, and putting the responsibility on the caller to check the clock
462 // status value being returned.
463 // TODO: this may not be the most ergonomic decision, putting a pin here to revisit this
464 // decision once the client code is fleshed out.
465 let clock_status = match self.clock_status {
466 // If the status in the shared memory segment is Unknown or Disrupted, returns that
467 // status.
468 ClockStatus::Unknown | ClockStatus::Disrupted => self.clock_status,
469
470 // If the status is Synchronized or FreeRunning, the expectation from the client is
471 // that the data is useable. However, if the clockbound daemon died or has not update
472 // the shared memory segment in a while, the status written to the shared memory
473 // segment may not be reliable anymore.
474 ClockStatus::Synchronized | ClockStatus::FreeRunning => {
475 if mono > self.void_after {
476 // The last update is old and beyond the horizon defined by the daemon, no
477 // guarantee is provided anymore, hence report Unknown status.
478 ClockStatus::Unknown
479 } else if mono > self.as_of + FREE_RUNNING_GRACE_PERIOD {
480 // The last update is too old to be trusted to be synchronized, reports Free
481 // Running status.
482 ClockStatus::FreeRunning
483 } else {
484 // The last update is recent enough, hence report it
485 self.clock_status
486 }
487 }
488 };
489
490 // Calculate the duration that has elapsed between the instant when the CEB parameters were
491 // snapshot'ed from the SHM segment (approximated by `as_of`), and the instant when the
492 // request to calculate the CEB was actually requested (approximated by `mono`). This
493 // duration is used to compute the growth of the error bound due to local dispersion
494 // between polling chrony and now.
495 //
496 // To avoid miscalculation in case the synchronization daemon is restarted, a
497 // CLOCK_MONOTONIC is used, since it is designed to not jump. Because we want this to be
498 // fast, and the exact accuracy is not critical here, we use CLOCK_MONOTONIC_COARSE on
499 // platforms that support it.
500 //
501 // But ... there is a catch. When validating causality of these events that is, `as_of`
502 // should always be older than `mono`, we observed this test to sometimes fail, with `mono`
503 // being older by a handful of nanoseconds. The root cause is not completely understood,
504 // but points to the clock resolution and/or update strategy and/or propagation of the
505 // updates through the VDSO memory page. See this for details:
506 // https://t.corp.amazon.com/P101954401.
507 //
508 // The following implementation is a mitigation.
509 // 1. if as_of <= mono is younger than as_of, calculate the duration (happy path)
510 // 2. if as_of - epsilon < mono < as_of, set the duration to 0
511 // 3. if mono < as_of - epsilon, return an error
512 //
513 // In short, this relaxes the sanity check a bit to accept some imprecision in the clock
514 // reading routines.
515 //
516 // What is a good value for `epsilon`?
517 // The CLOCK_MONOTONIC_COARSE resolution is a function of the HZ kernel variable defining
518 // the last kernel tick that drives this clock (e.g. HZ=250 leads to a 4 millisecond
519 // resolution). We could use the `clock_getres()` system call to retrieve this value but
520 // this makes diagnosing over different platform / OS configurations more complex. Instead
521 // settling on an arbitrary default value of 1 millisecond.
522 let causality_blur = self.as_of - TimeSpec::new(0, 1000);
523
524 let duration = if mono >= self.as_of {
525 // Happy path, no causality doubt
526 mono - self.as_of
527 } else if mono > causality_blur {
528 // Causality is "almost" broken. We are within a range that could be due to the clock
529 // precision. Let's approximate this to equality between mono and as_of.
530 TimeSpec::new(0, 0)
531 } else {
532 // Causality is breached.
533 return Err(ShmError::CausalityBreach(format!(
534 "as_of ({:?}) more recent than {:?}",
535 self.as_of, mono
536 )));
537 };
538
539 // Inflate the bound on clock error with the maximum drift the clock may be experiencing
540 // between the snapshot being read and ~now.
541 let duration_sec = duration.num_nanoseconds() as f64 / 1_000_000_000_f64;
542 let updated_bound = TimeSpec::nanoseconds(
543 self.bound_nsec + (duration_sec * f64::from(self.max_drift_ppb)) as i64,
544 );
545
546 // Build the (earliest, latest) interval within which true time exists.
547 let earliest = real - updated_bound;
548 let latest = real + updated_bound;
549
550 Ok(ClockBoundNowResult {
551 earliest,
552 latest,
553 clock_status,
554 })
555 }
556}
557
558impl ClockBoundSnapshot for ClockErrorBoundV2 {
559 /// The `ClockErrorBoundV2` implementation of `now()`.
560 ///
561 /// This version relies on the system clock to retrieve the current time as well as grow the
562 /// bound on the clock error at a constant rate.
563 fn now(&self) -> Result<ClockBoundNowResult, ShmError> {
564 // Read the clock, start with the REALTIME one to be as close as possible to the event the
565 // caller is interested in. The monotonic clock should be read after. It is correct for the
566 // process be preempted between the two calls: a delayed read of the monotonic clock will
567 // make the bound on clock error more pessimistic, but remains correct.
568 let real = clock_gettime_safe(CLOCK_REALTIME)?;
569 let mono = clock_gettime_safe(CLOCK_MONOTONIC)?;
570
571 self.compute_bound_at(real, mono)
572 }
573}
574
575/// Structure that holds the `ClockErrorBound` data captured at a specific point in time and valid
576/// until a subsequent point in time.
577///
578/// The `ClockErrorBound` structure supports calculating the actual bound on clock error at any time,
579/// using its `now()` method. The internal fields are not meant to be accessed directly.
580///
581/// Note that this version of the layout allow to not use the OS system clock to retrieve the
582/// current time or grow the clock error bound.
583///
584/// The structure is shared across the Shared Memory segment and has a C representation to enforce
585/// this specific layout.
586#[repr(C)]
587#[derive(Debug, Copy, Clone, PartialEq)]
588pub struct ClockErrorBoundV3 {
589 /// TSC counter value identifying this clock update.
590 ///
591 /// The TSC counter timestamp marking the time the clock and the clock error bound where
592 /// updated last. It represents the same instant as `as_of`.
593 as_of_tsc: u64,
594
595 /// Timestamp of this clock update.
596 ///
597 /// The nanosecond resolution timestamp marking the time the clock and the clock error bound
598 /// where updated last. This timestamp is derived from `as_of_tsc`.
599 as_of: TimeSpec,
600
601 /// Time after which this clock update is void.
602 ///
603 /// The nanosecond timestamp beyond which the bound on clock error should not be trusted. This
604 /// is a useful signal that the communication with the synchronization daemon is has failed,
605 /// for example.
606 void_after: TimeSpec,
607
608 /// Oscillator period estimate.
609 ///
610 /// The period of the oscillator in seconds per tick as a scaled fraction:
611 /// `period = period_frac / 2^(64 + period_shift)`.
612 period_frac: u64,
613
614 /// Oscillator period estimate error.
615 ///
616 /// The error on the period estimate as a dimensionless ratio.
617 /// The relative error `period_max_error / period` is similar to what the V2 layout folds into
618 /// `max_drift_ppb`, but as a scaled fraction:
619 /// `period_err = period_err_frac / 2^(64 + period_err_shift)`.
620 /// A client grows the bound by `elapsed × period_err`, exactly as it does with `max_drift_ppb`.
621 /// The value is always within [0, 1), or not written.
622 period_err_frac: u64,
623
624 /// Clock Error Bound
625 ///
626 /// An absolute upper bound on the accuracy of the feed-forward synchronization clock with
627 /// regards to true time at the instant represented by `as_of` and `as_of_tsc`.
628 bound_nsec: i64,
629
630 /// Disruption marker.
631 ///
632 /// This value is incremented (by an unspecified delta) each time the clock has been disrupted.
633 /// This count value is specific to a particular VM/EC2 instance.
634 disruption_marker: u64,
635
636 /// Maximum drift rate in part-per-billion.
637 ///
638 /// Maximum drift rate of the clock between updates of the synchronization daemon. The value
639 /// stored in `bound_nsec` should increase by the following to account for the clock drift
640 /// since `bound_nsec` was computed:
641 /// `bound_nsec += max_drift_ppb * (now - as_of)`
642 max_drift_ppb: u32,
643
644 /// Clock status.
645 ///
646 /// The synchronization daemon status indicates whether the daemon is synchronized,
647 /// free-running, etc.
648 clock_status: ClockStatus,
649
650 /// Clock disruption support enabled flag.
651 ///
652 /// This indicates whether or not the ClockBound daemon was started with a
653 /// configuration that supports detecting clock disruptions.
654 clock_disruption_support_enabled: bool,
655
656 /// Period shift
657 ///
658 /// The scaling of `period_frac`: the number of extra bits, beyond 64, the fraction is shifted
659 /// by so that a small period keeps its significant digits (see `PeriodFrac`).
660 period_shift: u8,
661
662 /// Period error shift
663 ///
664 /// The scaling of `period_err_frac`: the number of extra bits, beyond 64, the fraction is shifted
665 /// by so that a small value keeps its significant digits (see `PeriodFrac`).
666 period_err_shift: u8,
667
668 /// Padding.
669 _padding: [u8; 5],
670}
671
672impl ClockErrorBoundV3 {
673 /// Create a new `ClockErrorBound` struct.
674 #[allow(clippy::too_many_arguments)]
675 pub fn new(
676 as_of_tsc: u64,
677 as_of: TimeSpec,
678 void_after: TimeSpec,
679 period: f64,
680 period_err: f64,
681 bound_nsec: i64,
682 disruption_marker: u64,
683 max_drift_ppb: u32,
684 clock_status: ClockStatus,
685 clock_disruption_support_enabled: bool,
686 ) -> ClockErrorBoundV3 {
687 // Convert period and period_err into u64 representation
688 let p_frac = PeriodFrac::from(period);
689 let p_err_frac = PeriodFrac::from(period_err);
690
691 ClockErrorBoundV3 {
692 as_of_tsc,
693 as_of,
694 void_after,
695 period_frac: p_frac.frac,
696 period_err_frac: p_err_frac.frac,
697 bound_nsec,
698 disruption_marker,
699 max_drift_ppb,
700 clock_status,
701 clock_disruption_support_enabled,
702 period_shift: p_frac.shift,
703 period_err_shift: p_err_frac.shift,
704 _padding: [0u8; 5],
705 }
706 }
707
708 /// Get the oscillator period as a floating point value in seconds.
709 fn period(&self) -> f64 {
710 f64::from(PeriodFrac {
711 frac: self.period_frac,
712 shift: self.period_shift,
713 })
714 }
715
716 /// Get the oscillator period error as a floating point value.
717 fn period_err(&self) -> f64 {
718 f64::from(PeriodFrac {
719 frac: self.period_err_frac,
720 shift: self.period_err_shift,
721 })
722 }
723
724 #[allow(clippy::cast_precision_loss)]
725 #[allow(clippy::cast_possible_truncation)]
726 fn compute_bound_at_tsc(&self, now_tsc: u64) -> Result<ClockBoundNowResult, ShmError> {
727 // Sanity checks:
728 // - `now()` should operate on a consistent snapshot of the shared memory segment, and
729 // causality between mono and as_of should be enforced.
730 // - a extremely high value of the `max_drift_ppb` is a sign of something going wrong
731 if self.max_drift_ppb >= 1_000_000_000 {
732 return Err(ShmError::SegmentMalformed(format!(
733 "max_drift_ppb too large: [{}]",
734 self.max_drift_ppb,
735 )));
736 }
737
738 // Compute the number of TSC cycles between now and the instant the ff-sync clock was
739 // updated last. This is computed of a TSC value stored in the snapshot, hence this
740 // duration should never be negative.
741 let duration_tsc = now_tsc.saturating_sub(self.as_of_tsc);
742 let duration = duration_tsc as f64 * self.period();
743 let duration_nsec = duration_tsc as f64 * self.period() * NANOS_PER_SECOND;
744
745 // Convert the TSC timestamp into seconds with a linear projection.
746 let now = TimeSpec::nanoseconds(
747 self.as_of.tv_nsec()
748 + NANOS_PER_SECOND as i64 * self.as_of.tv_sec()
749 + duration_nsec as i64,
750 );
751
752 // Similarly, need to grow the bound on the clock error since the last update.
753 //
754 // First, amount for the underlying oscillator drifts (possibly at a worse
755 // possible rate) in between consecutive clock adjustments.
756 let oscillator_err_nsec = duration * f64::from(self.max_drift_ppb);
757 // And take into account the fact that the ff-sync period is an estimate (polluted by
758 // measurement noise): `period_err` is its relative error.
759 let p_estimate_err_nsec = duration_nsec * self.period_err();
760
761 let updated_bound = TimeSpec::nanoseconds(
762 (self.bound_nsec as f64 + oscillator_err_nsec + p_estimate_err_nsec) as i64,
763 );
764
765 // Build the (earliest, latest) interval within which true time exists.
766 let earliest = now - updated_bound;
767 let latest = now + updated_bound;
768
769 // If the ClockErrorBound data has not been updated "recently", the status of the clock
770 // cannot be guaranteed. Things are ambiguous, the synchronization daemon may be dead, or
771 // its interaction with the clockbound daemon is broken, or ... In any case, we signal the
772 // caller that guarantees are gone. We could return an Err here, but choosing to leverage
773 // ClockStatus instead, and putting the responsibility on the caller to check the clock
774 // status value being returned.
775 let clock_status = match self.clock_status {
776 // If the status in the shared memory segment is Unknown or Disrupted, returns that
777 // status.
778 ClockStatus::Unknown | ClockStatus::Disrupted => self.clock_status,
779
780 // If the status is Synchronized or FreeRunning, the expectation from the client is
781 // that the data is useable. However, if the clockbound daemon died or has not update
782 // the shared memory segment in a while, the status written to the shared memory
783 // segment may not be reliable anymore.
784 ClockStatus::Synchronized | ClockStatus::FreeRunning => {
785 if now > self.void_after {
786 // The last update is old and beyond the horizon defined by the daemon, no
787 // guarantee is provided anymore, hence report Unknown status.
788 ClockStatus::Unknown
789 } else if now > self.as_of + FREE_RUNNING_GRACE_PERIOD {
790 // The last update is too old to be trusted to be synchronized, reports Free
791 // Running status.
792 ClockStatus::FreeRunning
793 } else {
794 // The last update is recent enough, hence report it
795 self.clock_status
796 }
797 }
798 };
799
800 Ok(ClockBoundNowResult {
801 earliest,
802 latest,
803 clock_status,
804 })
805 }
806}
807
808impl ClockBoundSnapshot for ClockErrorBoundV3 {
809 /// The `ClockErrorBoundV3` implementation of `now()`.
810 ///
811 /// This version relies on the system clock to retrieve the current time as well as grow the
812 /// bound on the clock error at a constant rate.
813 fn now(&self) -> Result<ClockBoundNowResult, ShmError> {
814 let now_tsc = read_timestamp_counter_begin();
815 self.compute_bound_at_tsc(now_tsc)
816 }
817}
818
819/// A value in `[0, 1)` — a period in seconds per tick, or a relative error — as a scaled fraction
820/// `frac / 2^(64 + shift)`, the shift chosen so that a small value keeps its significant digits.
821struct PeriodFrac {
822 frac: u64,
823 shift: u8,
824}
825
826impl PeriodFrac {
827 /// Calculate the multiplication factor to maximize the number of significant digits when
828 /// converting the period from a floating point to an integer representation.
829 ///
830 /// # Panic:
831 /// Panic if the period passed is larger that 1 second.
832 ///
833 #[allow(clippy::cast_precision_loss)]
834 #[allow(clippy::cast_possible_truncation)]
835 #[allow(clippy::cast_sign_loss)]
836 fn calculate_frac_shift(period: f64) -> u8 {
837 // 1HZ and slower should not be seen.
838 assert!(
839 period < 1.0,
840 "Cannot convert period larger than 1 second: {period}"
841 );
842
843 // Protects against the case where a zero period is passed in.
844 if period == 0_f64 {
845 return 0_u8;
846 }
847 let freq: u64 = (1.0 / period) as u64;
848 // Cast: at most 64 zeros in a u64, hence can never go over u8::MAX.
849 (64 - freq.leading_zeros() - 1) as u8
850 }
851}
852
853impl From<f64> for PeriodFrac {
854 #[allow(clippy::cast_precision_loss)]
855 #[allow(clippy::cast_possible_truncation)]
856 #[allow(clippy::cast_sign_loss)]
857 fn from(value: f64) -> Self {
858 let shift = PeriodFrac::calculate_frac_shift(value);
859 // Cast: 64 + 255 unsigned does fit into a i32 without risk of sign error
860 let scale = 64 + i32::from(shift);
861 let frac = (value * 2_f64.powi(scale)) as u64;
862 PeriodFrac { frac, shift }
863 }
864}
865
866impl From<PeriodFrac> for f64 {
867 #[allow(clippy::cast_precision_loss)]
868 #[allow(clippy::cast_possible_truncation)]
869 fn from(value: PeriodFrac) -> Self {
870 let denominator = 2_f64.powi(64 + i32::from(value.shift));
871 (value.frac as f64) / denominator
872 }
873}
874
875#[cfg(test)]
876mod t_lib {
877 use super::*;
878
879 // Convenience macro to build ClockBoundError for unit tests
880 macro_rules! clockbound_v2 {
881 (($asof_tv_sec:literal, $asof_tv_nsec:literal), ($after_tv_sec:literal, $after_tv_nsec:literal)) => {
882 ClockErrorBoundV2::new(
883 TimeSpec::new($asof_tv_sec, $asof_tv_nsec), // as_of
884 TimeSpec::new($after_tv_sec, $after_tv_nsec), // void_after
885 10000, // bound_nsec
886 0, // disruption_marker
887 1000, // max_drift_ppb
888 ClockStatus::Synchronized, // clock_status
889 true, // clock_disruption_support_enabled
890 )
891 };
892 }
893
894 /// Assert the bound on clock error is computed correctly
895 #[test]
896 fn compute_bound_ok() {
897 let ceb = clockbound_v2!((0, 0), (10, 0));
898 let real = TimeSpec::new(2, 0);
899 let mono = TimeSpec::new(2, 0);
900
901 let ClockBoundNowResult {
902 earliest,
903 latest,
904 clock_status,
905 } = ceb
906 .compute_bound_at(real, mono)
907 .expect("Failed to compute bound");
908
909 // 2 seconds have passed since the bound was snapshot, hence 2 microsec of drift on top of
910 // the default 10 microsec put in the ClockBoundError data
911 assert_eq!(earliest.tv_sec(), 1);
912 assert_eq!(earliest.tv_nsec(), 1_000_000_000 - 12_000);
913 assert_eq!(latest.tv_sec(), 2);
914 assert_eq!(latest.tv_nsec(), 12_000);
915 assert_eq!(clock_status, ClockStatus::Synchronized);
916 }
917
918 /// Assert the bound on clock error is computed correctly, with realtime and monotonic clocks
919 /// disagreeing on time
920 #[test]
921 fn compute_bound_ok_when_real_ahead() {
922 let ceb = clockbound_v2!((0, 0), (10, 0));
923 let real = TimeSpec::new(20, 0); // realtime clock way ahead
924 let mono = TimeSpec::new(4, 0);
925
926 let ClockBoundNowResult {
927 earliest,
928 latest,
929 clock_status,
930 } = ceb
931 .compute_bound_at(real, mono)
932 .expect("Failed to compute bound");
933
934 // 4 seconds have passed since the bound was snapshot, hence 4 microsec of drift on top of
935 // the default 10 microsec put in the ClockBoundError data
936 assert_eq!(earliest.tv_sec(), 19);
937 assert_eq!(earliest.tv_nsec(), 1_000_000_000 - 14_000);
938 assert_eq!(latest.tv_sec(), 20);
939 assert_eq!(latest.tv_nsec(), 14_000);
940 assert_eq!(clock_status, ClockStatus::Synchronized);
941 }
942
943 /// Assert the clock status is FreeRunning if the ClockErrorBound data is passed the free
944 /// running grace period, simulating behavior of the daemon has died.
945 #[test]
946 fn compute_bound_force_free_running_status() {
947 let ceb = clockbound_v2!((0, 0), (100, 0));
948 let real = TimeSpec::new(61, 0);
949 let mono = TimeSpec::new(61, 0);
950
951 let ClockBoundNowResult {
952 earliest,
953 latest,
954 clock_status,
955 } = ceb
956 .compute_bound_at(real, mono)
957 .expect("Failed to compute bound");
958
959 // 61 seconds have passed since the bound was snapshot, hence 61 microsec of drift have
960 // accumulated at max_drift_ppb on top of the default 10 microsec put in the
961 // ClockBoundError data.
962 assert_eq!(earliest.tv_sec(), 60);
963 assert_eq!(earliest.tv_nsec(), 1_000_000_000 - 71_000);
964 assert_eq!(latest.tv_sec(), 61);
965 assert_eq!(latest.tv_nsec(), 71_000);
966 assert_eq!(clock_status, ClockStatus::FreeRunning);
967 }
968
969 /// Assert the clock status is Unknown if the ClockErrorBound data is passed void_after
970 #[test]
971 fn compute_bound_unknown_status_if_expired() {
972 let ceb = clockbound_v2!((0, 0), (5, 0));
973 let real = TimeSpec::new(10, 0);
974 let mono = TimeSpec::new(10, 0); // Passed void_after
975
976 let ClockBoundNowResult {
977 earliest,
978 latest,
979 clock_status,
980 } = ceb
981 .compute_bound_at(real, mono)
982 .expect("Failed to compute bound");
983
984 // 10 seconds have passed since the bound was snapshot, hence 10 microsec of drift on top of
985 // the default 10 microsec put in the ClockBoundError data
986 assert_eq!(earliest.tv_sec(), 9);
987 assert_eq!(earliest.tv_nsec(), 1_000_000_000 - 20_000);
988 assert_eq!(latest.tv_sec(), 10);
989 assert_eq!(latest.tv_nsec(), 20_000);
990 assert_eq!(clock_status, ClockStatus::Unknown);
991 }
992
993 /// Assert errors are returned if the ClockBoundError data is malformed with bad drift
994 #[test]
995 fn compute_bound_bad_drift() {
996 let mut ceb = clockbound_v2!((0, 0), (10, 0));
997 let real = TimeSpec::new(5, 0);
998 let mono = TimeSpec::new(5, 0);
999 ceb.max_drift_ppb = 2_000_000_000;
1000
1001 assert!(ceb.compute_bound_at(real, mono).is_err());
1002 }
1003
1004 /// Assert errors are returned if the ClockBoundError data snapshot has been taken after
1005 /// reading clocks at 'now'
1006 #[test]
1007 fn compute_bound_causality_break() {
1008 let ceb = clockbound_v2!((5, 0), (10, 0));
1009 let real = TimeSpec::new(1, 0);
1010 let mono = TimeSpec::new(1, 0);
1011
1012 let res = ceb.compute_bound_at(real, mono);
1013
1014 assert!(res.is_err());
1015 }
1016
1017 #[test]
1018 fn test_ceb_v3_new() {
1019 let ceb = ClockErrorBoundV3::new(
1020 1000, // as_of_tsc
1021 TimeSpec::new(1, 0), // as_of
1022 TimeSpec::new(10, 0), // void_after
1023 1e-9, // period
1024 1e-12, // period_err
1025 5000, // bound_nsec
1026 42, // disruption_marker
1027 1000, // max_drift_ppb
1028 ClockStatus::Synchronized, // clock_status
1029 true, // clock_disruption_support_enabled
1030 );
1031
1032 assert_eq!(ceb.as_of_tsc, 1000);
1033 assert_eq!(ceb.as_of, TimeSpec::new(1, 0));
1034 assert_eq!(ceb.void_after, TimeSpec::new(10, 0));
1035 assert_eq!(ceb.bound_nsec, 5000);
1036 assert_eq!(ceb.disruption_marker, 42);
1037 assert_eq!(ceb.max_drift_ppb, 1000);
1038 assert_eq!(ceb.clock_status, ClockStatus::Synchronized);
1039 assert_eq!(ceb.clock_disruption_support_enabled, true);
1040
1041 // Test period conversion
1042 let period = ceb.period();
1043 assert!((period - 1e-9).abs() < 1e-15);
1044
1045 let period_err = ceb.period_err();
1046 assert!((period_err - 1e-12).abs() < 1e-18);
1047 }
1048
1049 #[test]
1050 fn test_ceb_v3_period_conversion() {
1051 let ceb = ClockErrorBoundV3::new(
1052 0,
1053 TimeSpec::new(0, 0),
1054 TimeSpec::new(10, 0),
1055 2.5e-9, // 400 MHz
1056 1e-11,
1057 1000,
1058 0,
1059 1000,
1060 ClockStatus::Synchronized,
1061 true,
1062 );
1063
1064 let period = ceb.period();
1065 let relative_error = (period - 2.5e-9).abs() / 2.5e-9;
1066 assert!(relative_error < 1e-10);
1067
1068 let period_err = ceb.period_err();
1069 let relative_error = (period_err - 1e-11).abs() / 1e-11;
1070 assert!(relative_error < 1e-10);
1071 }
1072
1073 #[test]
1074 fn test_v3_compute_bound_at_tsc_synchronized_status() {
1075 // Create a V3 CEB with known values
1076 let ceb = ClockErrorBoundV3::new(
1077 1_000_000_000, // as_of_tsc (1 billion cycles)
1078 TimeSpec::new(1, 0), // as_of = 1 second
1079 TimeSpec::new(100, 0), // void_after = 100 seconds
1080 1e-9, // period = 1 ns (1 GHz clock)
1081 1e-12, // period_err = 1ps
1082 10_000, // bound_nsec = 10 microseconds
1083 0, // disruption_marker
1084 1000, // max_drift_ppb = 1 ppm
1085 ClockStatus::Synchronized,
1086 true,
1087 );
1088
1089 // Simulate reading TSC 2 seconds later (2 billion more cycles at 1 GHz)
1090 let now_tsc = 3_500_000_000;
1091
1092 let result = ceb.compute_bound_at_tsc(now_tsc).expect("Should succeed");
1093
1094 // Expected time: as_of + 2 seconds = 3 seconds
1095 assert_eq!(result.earliest.tv_sec(), 3); // approximately
1096 assert_eq!(result.latest.tv_sec(), 3); // approximately
1097
1098 // Status should still be Synchronized (within grace period)
1099 assert_eq!(result.clock_status, ClockStatus::Synchronized);
1100 }
1101
1102 /// The V3 bound grows by both rates, to the nanosecond: the oscillator's worst-case drift
1103 /// (`max_drift_ppb`) and the period estimate's relative error (`period_err`), each times the
1104 /// elapsed time.
1105 #[test]
1106 fn test_ceb_v3_bound_grows_with_drift_and_period_error() {
1107 let ceb = ClockErrorBoundV3::new(
1108 0, // as_of_tsc
1109 TimeSpec::new(1, 0), // as_of
1110 TimeSpec::new(100, 0), // void_after
1111 1e-9, // period = 1 ns (1 GHz clock)
1112 1e-6, // period_err = 1 ppm, relative
1113 10_000, // bound_nsec = 10 µs
1114 0, // disruption_marker
1115 2_000, // max_drift_ppb = 2 ppm
1116 ClockStatus::Synchronized,
1117 true,
1118 );
1119
1120 // 2 s later (2 billion ticks): drift 2 ppm × 2 s = 4 µs; period error 1 ppm × 2 s = 2 µs.
1121 let result = ceb
1122 .compute_bound_at_tsc(2_000_000_000)
1123 .expect("Should succeed");
1124 assert_eq!(
1125 result.earliest,
1126 TimeSpec::new(3, 0) - TimeSpec::nanoseconds(16_000)
1127 );
1128 assert_eq!(
1129 result.latest,
1130 TimeSpec::new(3, 0) + TimeSpec::nanoseconds(16_000)
1131 );
1132 }
1133
1134 // Assert that typical TSC periods (1Hz to 10 GHz range) are converted into scaled integers
1135 // without a loss of precision.
1136 #[test]
1137 fn test_period_frac_conversion_typical_periods() {
1138 let periods = [1e-3, 1e-6, 1e-7, 1e-8, 1e-9, 2e-9, 5e-9, 1e-10];
1139
1140 for &period in &periods {
1141 let frac = PeriodFrac::from(period);
1142 let result: f64 = f64::from(frac);
1143 assert!(result == period);
1144 }
1145 }
1146
1147 // Assert atypical TSC periods are converted into scaled integers
1148 // with a minimum loss of precision.
1149 #[test]
1150 fn test_period_frac_conversion_edge_cases() {
1151 // Very small period (very high frequency)
1152 let small_period = 1e-25;
1153 let frac = PeriodFrac::from(small_period);
1154 let result: f64 = f64::from(frac);
1155 let relative_error = (result - small_period).abs() / small_period;
1156 assert!(relative_error < 1e-10);
1157
1158 // Larger period (lower frequency)
1159 let large_period = 0.1;
1160 let frac = PeriodFrac::from(large_period);
1161 let result: f64 = f64::from(frac);
1162 let relative_error = (result - large_period).abs() / large_period;
1163 assert!(relative_error < 1e-10);
1164 }
1165
1166 // Assert that the conversion panics on non-realistic frequencies.
1167 #[test]
1168 #[should_panic(expected = "Cannot convert period larger than 1 second")]
1169 fn test_period_frac_conversion_panci() {
1170 let large_period = 1.0;
1171 let _ = PeriodFrac::from(large_period);
1172 }
1173
1174 #[test]
1175 fn test_calculate_frac_shift_typical() {
1176 // For a 1 GHz clock (period = 1e-9), frequency = 1e9
1177 // 1e9 in binary is about 30 bits, so shift should be around 29
1178 let period = 1e-9;
1179 let shift = PeriodFrac::calculate_frac_shift(period);
1180 assert!(shift >= 29 && shift <= 30, "shift = {}", shift);
1181
1182 // For a 2.5 GHz clock (period = 4e-10), frequency = 2.5e9
1183 // 2.5e9 in binary is about 31 bits
1184 let period = 4e-10;
1185 let shift = PeriodFrac::calculate_frac_shift(period);
1186 assert!(shift >= 30 && shift <= 32, "shift = {}", shift);
1187 }
1188
1189 #[test]
1190 fn test_calculate_frac_shift_zero() {
1191 // Zero period should return 0 (max of 0 and negative value)
1192 let period = 0.0;
1193 let shift = PeriodFrac::calculate_frac_shift(period);
1194 assert_eq!(shift, 0);
1195 }
1196
1197 #[test]
1198 fn test_zero_period_frac_conversion() {
1199 // Test that zero period doesn't panic and gives reasonable result
1200 let period = 0.0;
1201 let frac = PeriodFrac::from(period);
1202 assert_eq!(frac.shift, 0);
1203 assert_eq!(frac.frac, 0);
1204
1205 let result: f64 = f64::from(frac);
1206 assert_eq!(result, 0.0);
1207 }
1208
1209 #[test]
1210 fn test_precision_maintained() {
1211 // Test that we maintain good precision across conversions
1212 let period = 2.718281828e-9; // Some arbitrary value
1213 let frac = PeriodFrac::from(period);
1214 let result: f64 = f64::from(frac);
1215
1216 // Should maintain at least 10 significant digits
1217 let relative_error = (result - period).abs() / period;
1218 assert!(relative_error < 1e-10);
1219 }
1220
1221 #[test]
1222 fn test_frac_representation_property() {
1223 // Test that the fixed-point representation makes sense
1224 let period = 1e-9;
1225 let frac = PeriodFrac::from(period);
1226
1227 // frac should be non-zero for non-zero period
1228 assert!(frac.frac > 0);
1229
1230 // shift should be reasonable (not 0 or 255)
1231 assert!(frac.shift > 0 && frac.shift < 64);
1232 }
1233}