lightning_rapid_gossip_sync/
lib.rs

1#![deny(rustdoc::broken_intra_doc_links)]
2#![deny(rustdoc::private_intra_doc_links)]
3#![deny(missing_docs)]
4#![deny(unsafe_code)]
5#![deny(non_upper_case_globals)]
6#![deny(non_camel_case_types)]
7#![deny(non_snake_case)]
8#![deny(unused_mut)]
9#![deny(unused_variables)]
10#![deny(unused_imports)]
11//! This crate exposes client functionality to rapidly sync gossip data, aimed primarily at mobile
12//! devices.
13//!
14//! The rapid gossip sync server will provide a compressed response containing differential gossip
15//! data. The gossip data is formatted compactly, omitting signatures and opportunistically
16//! incremental where previous channel updates are known. This mechanism is enabled when the
17//! timestamp of the last known channel update is communicated. A reference server implementation
18//! can be found [on Github](https://github.com/lightningdevkit/rapid-gossip-sync-server).
19//!
20//! The primary benefit of this syncing mechanism is that it allows a low-powered client to offload
21//! the validation of gossip signatures to a semi-trusted server. This enables the client to
22//! privately calculate routes for payments, and to do so much faster than requiring a full
23//! peer-to-peer gossip sync to complete.
24//!
25//! The server calculates its response on the basis of a client-provided `latest_seen` timestamp,
26//! i.e., the server will return all rapid gossip sync data it has seen after the given timestamp.
27//!
28//! # Getting Started
29//! Firstly, the data needs to be retrieved from the server. For example, you could use the server
30//! at <https://rapidsync.lightningdevkit.org> with the following request format:
31//!
32//! ```shell
33//! curl -o rapid_sync.lngossip https://rapidsync.lightningdevkit.org/snapshot/<last_sync_timestamp>
34//! ```
35//! Note that the first ever rapid sync should use `0` for `last_sync_timestamp`.
36//!
37//! After the gossip data snapshot has been downloaded, one of the client's graph processing
38//! functions needs to be called.
39#![cfg_attr(
40	feature = "std",
41	doc = "In this example, we process the update by reading its contents from disk, which we do by calling [`RapidGossipSync::update_network_graph`]:"
42)]
43#![cfg_attr(
44	not(feature = "std"),
45	doc = "In this example, we process the update by reading its contents from disk, which we do by calling [`RapidGossipSync::update_network_graph_no_std`]:"
46)]
47//!
48//! ```
49//! use bitcoin::constants::genesis_block;
50//! use bitcoin::Network;
51//! use lightning::routing::gossip::NetworkGraph;
52//! use lightning_rapid_gossip_sync::RapidGossipSync;
53//!
54//! # use lightning::util::logger::{Logger, Record};
55//! # struct FakeLogger {}
56//! # impl Logger for FakeLogger {
57//! #     fn log(&self, record: Record) { }
58//! # }
59//! # let logger = FakeLogger {};
60//!
61//! let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
62//! let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
63//! let snapshot_contents: &[u8] = &[0; 0];
64//! // In non-`std` environments you need to provide the current time in unix epoch seconds
65//! // otherwise you can use `update_network_graph`:
66#![cfg_attr(
67	feature = "std",
68	doc = "let new_last_sync_timestamp_result = rapid_sync.update_network_graph(snapshot_contents);"
69)]
70#![cfg_attr(not(feature = "std"), doc = "let current_time_unix = 0;")]
71#![cfg_attr(
72	not(feature = "std"),
73	doc = "let new_last_sync_timestamp_result = rapid_sync.update_network_graph_no_std(snapshot_contents, Some(current_time_unix));"
74)]
75//! ```
76
77#![cfg_attr(all(not(feature = "std"), not(test)), no_std)]
78
79#[cfg(ldk_bench)]
80extern crate criterion;
81
82#[cfg(not(feature = "std"))]
83extern crate alloc;
84
85use core::ops::Deref;
86use core::sync::atomic::{AtomicBool, Ordering};
87use lightning::io;
88use lightning::ln::msgs::{DecodeError, LightningError};
89use lightning::routing::gossip::NetworkGraph;
90use lightning::util::logger::Logger;
91
92/// Core functionality of this crate
93mod processing;
94
95/// All-encompassing standard error type that processing can return
96#[derive(Debug)]
97pub enum GraphSyncError {
98	/// Error trying to read the update data, typically due to an erroneous data length indication
99	/// that is greater than the actual amount of data provided
100	DecodeError(DecodeError),
101	/// Error applying the patch to the network graph, usually the result of updates that are too
102	/// old or missing prerequisite data to the application of updates out of order
103	LightningError(LightningError),
104}
105
106impl From<io::Error> for GraphSyncError {
107	fn from(error: io::Error) -> Self {
108		Self::DecodeError(DecodeError::Io(error.kind()))
109	}
110}
111
112impl From<bitcoin::secp256k1::Error> for GraphSyncError {
113	fn from(_: bitcoin::secp256k1::Error) -> Self {
114		Self::DecodeError(DecodeError::InvalidValue)
115	}
116}
117
118impl From<DecodeError> for GraphSyncError {
119	fn from(error: DecodeError) -> Self {
120		Self::DecodeError(error)
121	}
122}
123
124impl From<LightningError> for GraphSyncError {
125	fn from(error: LightningError) -> Self {
126		Self::LightningError(error)
127	}
128}
129
130/// The main Rapid Gossip Sync object.
131///
132/// See [crate-level documentation] for usage.
133///
134/// [crate-level documentation]: crate
135pub struct RapidGossipSync<NG: Deref<Target = NetworkGraph<L>>, L: Deref>
136where
137	L::Target: Logger,
138{
139	network_graph: NG,
140	logger: L,
141	is_initial_sync_complete: AtomicBool,
142}
143
144impl<NG: Deref<Target = NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L>
145where
146	L::Target: Logger,
147{
148	/// Instantiate a new [`RapidGossipSync`] instance.
149	pub fn new(network_graph: NG, logger: L) -> Self {
150		Self { network_graph, logger, is_initial_sync_complete: AtomicBool::new(false) }
151	}
152
153	/// Sync gossip data from a file.
154	/// Returns the last sync timestamp to be used the next time rapid sync data is queried.
155	///
156	/// `network_graph`: The network graph to apply the updates to
157	///
158	/// `sync_path`: Path to the file where the gossip update data is located
159	///
160	#[cfg(feature = "std")]
161	pub fn sync_network_graph_with_file_path(
162		&self, sync_path: &str,
163	) -> Result<u32, GraphSyncError> {
164		let file = std::fs::File::open(sync_path).map_err(|e| {
165			let bitcoin_error: lightning::io::Error = e.into();
166			bitcoin_error
167		})?;
168		let mut buf_reader = std::io::BufReader::new(file);
169		self.update_network_graph_from_byte_stream(&mut buf_reader)
170	}
171
172	/// Update network graph from binary data.
173	/// Returns the last sync timestamp to be used the next time rapid sync data is queried.
174	///
175	/// `update_data`: `&[u8]` binary stream that comprises the update data
176	#[cfg(feature = "std")]
177	pub fn update_network_graph(&self, update_data: &[u8]) -> Result<u32, GraphSyncError> {
178		let mut read_cursor = io::Cursor::new(update_data);
179		self.update_network_graph_from_byte_stream(&mut read_cursor)
180	}
181
182	/// Update network graph from binary data.
183	/// Returns the last sync timestamp to be used the next time rapid sync data is queried.
184	///
185	/// `update_data`: `&[u8]` binary stream that comprises the update data
186	/// `current_time_unix`: `Option<u64>` optional current timestamp to verify data age
187	pub fn update_network_graph_no_std(
188		&self, update_data: &[u8], current_time_unix: Option<u64>,
189	) -> Result<u32, GraphSyncError> {
190		let mut read_cursor = io::Cursor::new(update_data);
191		self.update_network_graph_from_byte_stream_no_std(&mut read_cursor, current_time_unix)
192	}
193
194	/// Gets a reference to the underlying [`NetworkGraph`] which was provided in
195	/// [`RapidGossipSync::new`].
196	///
197	/// This is not exported to bindings users as bindings don't support a reference-to-a-reference yet
198	pub fn network_graph(&self) -> &NG {
199		&self.network_graph
200	}
201
202	/// Returns whether a rapid gossip sync has completed at least once.
203	pub fn is_initial_sync_complete(&self) -> bool {
204		self.is_initial_sync_complete.load(Ordering::Acquire)
205	}
206}
207
208#[cfg(feature = "std")]
209#[cfg(test)]
210mod tests {
211	use std::fs;
212
213	use bitcoin::Network;
214
215	use crate::{GraphSyncError, RapidGossipSync};
216	use lightning::ln::msgs::DecodeError;
217	use lightning::routing::gossip::NetworkGraph;
218	use lightning::util::test_utils::TestLogger;
219
220	#[test]
221	fn test_sync_from_file() {
222		struct FileSyncTest {
223			directory: String,
224		}
225
226		impl FileSyncTest {
227			fn new(tmp_directory: &str, valid_response: &[u8]) -> FileSyncTest {
228				let test = FileSyncTest { directory: tmp_directory.to_owned() };
229
230				let graph_sync_test_directory = test.get_test_directory();
231				fs::create_dir_all(graph_sync_test_directory).unwrap();
232
233				let graph_sync_test_file = test.get_test_file_path();
234				fs::write(graph_sync_test_file, valid_response).unwrap();
235
236				test
237			}
238
239			fn get_test_directory(&self) -> String {
240				self.directory.clone() + "/graph-sync-tests"
241			}
242
243			fn get_test_file_path(&self) -> String {
244				self.get_test_directory() + "/test_data.lngossip"
245			}
246		}
247
248		impl Drop for FileSyncTest {
249			fn drop(&mut self) {
250				fs::remove_dir_all(self.directory.clone()).unwrap();
251			}
252		}
253
254		// same as incremental_only_update_fails_without_prior_same_direction_updates
255		let valid_response = vec![
256			76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
257			79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
258			0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
259			187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
260			157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
261			88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
262			204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
263			181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
264			110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
265			76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
266			226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
267			0, 0, 0, 1, 0, 0, 0, 0, 58, 85, 116, 216, 255, 8, 153, 192, 0, 2, 27, 0, 0, 25, 0, 0,
268			0, 1, 0, 0, 0, 125, 255, 2, 68, 226, 0, 6, 11, 0, 1, 5, 0, 0, 0, 0, 29, 129, 25, 192,
269		];
270
271		let tmp_directory = "./rapid-gossip-sync-tests-tmp";
272		let sync_test = FileSyncTest::new(tmp_directory, &valid_response);
273		let graph_sync_test_file = sync_test.get_test_file_path();
274
275		let logger = TestLogger::new();
276		let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
277
278		assert_eq!(network_graph.read_only().channels().len(), 0);
279
280		let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
281		let sync_result = rapid_sync.sync_network_graph_with_file_path(&graph_sync_test_file);
282
283		if sync_result.is_err() {
284			panic!("Unexpected sync result: {:?}", sync_result)
285		}
286
287		assert_eq!(network_graph.read_only().channels().len(), 2);
288		let after = network_graph.to_string();
289		assert!(
290			after.contains("021607cfce19a4c5e7e6e738663dfafbbbac262e4ff76c2c9b30dbeefc35c00643")
291		);
292		assert!(
293			after.contains("02247d9db0dfafea745ef8c9e161eb322f73ac3f8858d8730b6fd97254747ce76b")
294		);
295		assert!(
296			after.contains("029e01f279986acc83ba235d46d80aede0b7595f410353b93a8ab540bb677f4432")
297		);
298		assert!(
299			after.contains("02c913118a8895b9e29c89af6e20ed00d95a1f64e4952edbafa84d048f26804c61")
300		);
301		assert!(after.contains("619737530008010752"));
302		assert!(after.contains("783241506229452801"));
303	}
304
305	#[test]
306	fn measure_native_read_from_file() {
307		let logger = TestLogger::new();
308		let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
309
310		assert_eq!(network_graph.read_only().channels().len(), 0);
311
312		let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
313		let start = std::time::Instant::now();
314		let sync_result = rapid_sync.sync_network_graph_with_file_path("./res/full_graph.lngossip");
315		if let Err(GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
316			let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-285cb27df79-2022-07-21.bin\n\n{:?}", io_error);
317			#[cfg(not(require_route_graph_test))]
318			{
319				println!("{}", error_string);
320				return;
321			}
322			#[cfg(require_route_graph_test)]
323			panic!("{}", error_string);
324		}
325		let elapsed = start.elapsed();
326		println!("initialization duration: {:?}", elapsed);
327		if sync_result.is_err() {
328			panic!("Unexpected sync result: {:?}", sync_result)
329		}
330	}
331}
332
333#[cfg(ldk_bench)]
334/// Benches
335pub mod bench {
336	use bitcoin::Network;
337
338	use criterion::Criterion;
339
340	use std::fs;
341
342	use lightning::routing::gossip::NetworkGraph;
343	use lightning::util::test_utils::TestLogger;
344
345	use crate::RapidGossipSync;
346
347	/// Bench!
348	pub fn bench_reading_full_graph_from_file(b: &mut Criterion) {
349		let logger = TestLogger::new();
350		b.bench_function("read_full_graph_from_rgs", |b| b.iter(|| {
351			let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
352			let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
353			let mut file = match fs::read("../lightning-rapid-gossip-sync/res/full_graph.lngossip") {
354				Ok(f) => f,
355				Err(io_error) => {
356					let error_string = format!(
357						"Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}",
358						io_error);
359					#[cfg(not(require_route_graph_test))]
360					{
361						println!("{}", error_string);
362						return;
363					}
364					#[cfg(require_route_graph_test)]
365					panic!("{}", error_string);
366				},
367			};
368			rapid_sync.update_network_graph_no_std(&mut file, None).unwrap();
369		}));
370	}
371}