vaegram 0.2.0

Telegram MTProto client library for users and bots
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
use std::collections::HashMap;
use std::future::{Future, poll_fn};
use std::ops::ControlFlow;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::task::Poll;
use std::time::Duration;

use super::internals::data_center::Dc;
use super::{Events, Known, Salt, Session, WeakSession, init_connection};
use crate::client::quiet;
use crate::connection::Options;
use crate::crypto::rsa::PublicKey;
use crate::errors::{Error, Result};
use crate::raw::{RemoteCall, base, functions};

const IDLE: Duration = Duration::from_secs(15);
// Telegram's largest part, both ways.
const MAX_PART: u64 = 512 << 10;
// Uploads keep two parts in flight per connection (downloads one), so a part can wait behind another.
const PARTS_PER_CONNECTION: u64 = 2;
// The slowest link a transfer still survives, in bytes per second per connection.
const MIN_RATE: u64 = 8 << 10;
const PART_TIMEOUT: Duration = Duration::from_secs(MAX_PART * PARTS_PER_CONNECTION / MIN_RATE);
const MAX_TIMEOUTS: usize = 3;
// A CDN gets at most this per wait, so one that keeps flooding is soon given up on and the master DC serves the part.
const MAX_CDN_FLOOD_WAIT: Duration = Duration::from_secs(60);

// Telegram shares a DC's auth key between its main and media-only addresses, so the key is kept per DC and the
// connections per address.
#[derive(Default)]
pub(crate) struct TransferPool {
    slots: Mutex<HashMap<(i32, bool), Arc<Slot>>>,
    // Goes up on each `close_all`, so a lease still opening connections knows to close them.
    epoch: AtomicU64,
    // Set by `shut`: no lease opens anything any more.
    shut: AtomicBool,
}

#[derive(Default)]
struct Slot {
    opening: tokio::sync::Mutex<()>,
    state: Mutex<State>,
}

#[derive(Default)]
struct State {
    auth_key: Option<[u8; 256]>,
    key_made: Option<tokio::time::Instant>,
    // The salt a connection on this key last had, so a new one doesn't start from 0 and bounce its first parts.
    salt: Salt,
    sessions: HashMap<Dc, Vec<WeakSession>>,
    // Numbers handed out so far, so an observer can tell this DC's connections apart.
    opened: u32,
}

#[derive(Clone, Copy)]
pub(crate) enum Auth<'a> {
    Home(&'a Session, i32),
    Cdn(&'a PublicKey),
}

pub(crate) struct Lease {
    pub(crate) sessions: Vec<Session>,
}

impl TransferPool {
    pub(crate) fn clear(&self) {
        self.slots.lock().unwrap().clear();
    }

    pub(crate) fn forget(&self, dc: Dc) {
        self.slots.lock().unwrap().remove(&(dc.id, dc.test));
    }

    /// Closes every file connection, also the ones a lease is still opening, and forgets the keys, so the next
    /// lease starts over.
    pub(crate) fn close_all(&self) {
        // Bumped under the map lock, so a lease either gets a slot this drains or sees the new epoch.
        let slots: Vec<Arc<Slot>> = {
            let mut map = self.slots.lock().unwrap();
            self.epoch.fetch_add(1, Ordering::AcqRel);
            map.drain().map(|(_, slot)| slot).collect()
        };
        for slot in slots {
            let state = slot.state.lock().unwrap();
            state.sessions.values().flatten().filter_map(WeakSession::upgrade).for_each(|s| s.close());
        }
    }

    /// `close_all` for good: later leases fail with `Error::Disconnected`.
    pub(crate) fn shut(&self) {
        self.shut.store(true, Ordering::Release);
        self.close_all();
    }

    #[cfg(test)]
    pub(crate) fn insert(&self, dc: Dc, session: &Session) {
        let slot = self.slots.lock().unwrap().entry((dc.id, dc.test)).or_default().clone();
        let mut state = slot.state.lock().unwrap();
        (state.auth_key, state.key_made, state.salt) = (Some(*session.auth_key()), session.key_made, session.salt());
        state.sessions.entry(dc).or_default().push(session.downgrade());
    }

    pub(crate) async fn lease(
        &self,
        dc: Dc,
        count: usize,
        auth: Auth<'_>,
        options: &Options,
        events: &Arc<Events>,
    ) -> Result<Lease> {
        // CDN DCs keep no permanent key to bind a temporary one to.
        let rebind = || match auth {
            Auth::Home(home, _) => home.pfs.clone(),
            Auth::Cdn(_) => None,
        };
        if self.shut.load(Ordering::Acquire) {
            return Err(Error::Disconnected);
        }
        let (epoch, slot) = {
            let mut map = self.slots.lock().unwrap();
            (self.epoch.load(Ordering::Acquire), map.entry((dc.id, dc.test)).or_default().clone())
        };
        let _opening = slot.opening.lock().await;
        let number = || {
            let mut state = slot.state.lock().unwrap();
            state.opened += 1;
            state.opened
        };
        let (mut sessions, key) = {
            let state = slot.state.lock().unwrap();
            let live: Vec<Session> =
                state.sessions.get(&dc).into_iter().flatten().filter_map(WeakSession::upgrade).collect();
            let home = match auth {
                Auth::Home(home, _) if home.dc_id() == dc.id => Some(home.known_key()),
                _ => None,
            };
            let salt = live.first().map_or(state.salt, Session::salt);
            (live, home.or(state.auth_key.map(|key| Known { key, salt, made: state.key_made })))
        };
        let key = match key {
            Some(key) => key,
            None => {
                let cdn_key = match auth {
                    Auth::Cdn(key) => Some(key),
                    Auth::Home(..) => None,
                };
                let session =
                    Session::connect_transfer(dc, None, cdn_key, options, events.clone(), rebind(), number()).await?;
                if let Auth::Home(home, api_id) = auth {
                    let export = functions::auth::ExportAuthorization { dc_id: dc.id };
                    // Same as everything else this client sends, when it wants no updates.
                    let base::auth::ExportedAuthorization::ExportedAuthorization(e) = match options.no_updates {
                        true => home.invoke(&quiet(export)).await?,
                        false => home.invoke(&export).await?,
                    };
                    let import = functions::auth::ImportAuthorization { id: e.id, bytes: e.bytes };
                    match options.no_updates {
                        true => session.invoke(&init_connection(api_id, options, quiet(import))).await?,
                        false => session.invoke(&init_connection(api_id, options, import)).await?,
                    };
                }
                let key = session.known_key();
                let mut state = slot.state.lock().unwrap();
                (state.auth_key, state.key_made) = (Some(key.key), key.made);
                drop(state);
                sessions.push(session);
                key
            }
        };
        let missing = count.saturating_sub(sessions.len());
        let connect = |_| Session::connect_transfer(dc, Some(key), None, options, events.clone(), rebind(), number());
        let opened = try_join_all((0..missing).map(connect)).await?;
        sessions.extend(opened);
        let mut state = slot.state.lock().unwrap();
        // Checked under the lock close_all takes after its bump: either it sees these sessions, or this sees the bump.
        if self.shut.load(Ordering::Acquire) || self.epoch.load(Ordering::Acquire) != epoch {
            drop(state);
            sessions.iter().for_each(Session::close);
            return Err(Error::Disconnected);
        }
        state.salt = sessions.first().map_or(key.salt, Session::salt);
        state.sessions.insert(dc, sessions.iter().map(Session::downgrade).collect());
        drop(state);
        sessions.truncate(count);
        Ok(Lease { sessions })
    }
}

impl Drop for Lease {
    fn drop(&mut self) {
        let sessions = std::mem::take(&mut self.sessions);
        if let Ok(runtime) = tokio::runtime::Handle::try_current() {
            runtime.spawn(async move {
                tokio::time::sleep(IDLE).await;
                drop(sessions);
            });
        }
    }
}

pub(crate) async fn invoke_part<R: RemoteCall>(session: &Session, request: &R) -> Result<R::Return> {
    invoke_part_on(session, request, false).await
}

pub(crate) async fn invoke_cdn_part<R: RemoteCall>(session: &Session, request: &R) -> Result<R::Return> {
    invoke_part_on(session, request, true).await
}

async fn invoke_part_on<R: RemoteCall>(session: &Session, request: &R, cdn: bool) -> Result<R::Return> {
    // Parts go straight to their session, so the user's request_timeout still has to count.
    let timeout = PART_TIMEOUT.max(session.options().request_timeout);
    let cap = cdn.then_some(MAX_CDN_FLOOD_WAIT);
    let (mut timeouts, mut retries, mut slept) = (0, 0u32, Duration::ZERO);
    loop {
        let result = session.invoke_with_timeout(request, timeout).await;
        let retry = result.as_ref().err().map(|e| session.should_retry(R::NAME, e, retries + 1, slept, cap));
        match (&result, retry) {
            (_, Some(ControlFlow::Continue(wait))) => {
                retries += 1;
                slept += wait;
                tokio::time::sleep(wait).await;
            }
            // Timeouts get their own retries whatever the policy thinks of them.
            (Err(Error::Timeout), _) if timeouts + 1 < MAX_TIMEOUTS => timeouts += 1,
            _ => return result,
        }
    }
}

pub(crate) fn workers(parts: i32, part_size: u64, budget: u64) -> usize {
    ((budget / part_size) as usize).clamp(1, parts.max(1) as usize)
}

pub(crate) fn take(next: &AtomicI32, count: i32) -> Option<i32> {
    let part = next.fetch_add(1, Ordering::Relaxed);
    (part < count).then_some(part)
}

pub(crate) type Report<'a> = Box<dyn FnMut(u64, u64) + Send + 'a>;

// The callback runs under the lock, so `done` only grows even when parts finish out of order.
pub(crate) struct Progress<'a> {
    total: u64,
    state: Mutex<(u64, Option<Report<'a>>)>,
}

impl<'a> Progress<'a> {
    pub(crate) fn new(report: Option<Report<'a>>, total: u64) -> Self {
        Progress { total, state: Mutex::new((0, report)) }
    }

    pub(crate) fn add(&self, bytes: usize) {
        let (done, report) = &mut *self.state.lock().unwrap();
        *done += bytes as u64;
        if let Some(report) = report {
            report(*done, self.total);
        }
    }
}

pub(crate) async fn try_join_all<T, F: Future<Output = Result<T>>>(futures: impl Iterator<Item = F>) -> Result<Vec<T>> {
    let mut futures: Vec<_> = futures.map(|f| Some(Box::pin(f))).collect();
    let mut outputs: Vec<Option<T>> = futures.iter().map(|_| None).collect();
    poll_fn(|cx| {
        let mut pending = false;
        for (slot, output) in futures.iter_mut().zip(&mut outputs) {
            let Some(future) = slot else { continue };
            match future.as_mut().poll(cx) {
                Poll::Ready(Ok(value)) => (*output, *slot) = (Some(value), None),
                Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
                Poll::Pending => pending = true,
            }
        }
        if pending { Poll::Pending } else { Poll::Ready(Ok(())) }
    })
    .await?;
    Ok(outputs.into_iter().flatten().collect())
}

#[cfg(test)]
mod tests {
    use std::ops::ControlFlow;
    use std::sync::Arc;
    use std::sync::Mutex;
    use std::sync::atomic::AtomicI32;
    use std::time::Duration;

    use super::{
        Auth, MAX_CDN_FLOOD_WAIT, PART_TIMEOUT, Progress, Session, TransferPool, invoke_cdn_part, invoke_part, take,
        try_join_all, workers,
    };
    use crate::Client;
    use crate::connection::{Options, Transport};
    use crate::errors::{Error, Result, RpcError};
    use crate::raw::{Serializable, base, functions, types};
    use crate::session::internals::data_center::Dc;
    use crate::session::test_dc::TestDc;
    use crate::session::{AutoSleep, Retry, RetryContext, RetryPolicy};

    #[tokio::test]
    async fn a_dcs_media_address_reuses_the_key_of_its_main_address() {
        let client = Client::offline();
        // Nothing listens on these ports, so a lease that tried to open a connection would fail.
        let dc = |port| Dc { id: 4, addr: ([127, 0, 0, 1], port).into(), test: false };
        let (foreign, _sent) = Session::scripted(Vec::new());
        let pool = TransferPool::default();
        pool.insert(dc(1), &foreign);
        let home = client.session();
        let lease = pool.lease(dc(2), 0, Auth::Home(&home, 1), &client.0.options, &client.0.events).await;
        assert!(lease.is_ok_and(|l| l.sessions.is_empty()));
    }

    #[test]
    fn caps_parts_in_flight_by_bytes() {
        let w = |parts, budget| workers(parts, 512 * 1024, budget);
        assert_eq!((w(1, 4 << 20), w(3, 4 << 20), w(100, 4 << 20), w(100, 2 << 20), w(0, 2 << 20)), (1, 3, 8, 4, 1));
    }

    #[test]
    fn reports_growing_progress_from_parallel_parts() {
        let mut seen = Vec::new();
        let progress = Progress::new(Some(Box::new(|done, total| seen.push((done, total)))), 2400);
        std::thread::scope(|s| {
            for _ in 0..8 {
                s.spawn(|| (0..100).for_each(|i| progress.add(1 + i % 5)));
            }
        });
        drop(progress);
        assert_eq!((seen.len(), seen.last().copied()), (800, Some((2400, 2400))));
        assert!(seen.windows(2).all(|w| w[0].0 < w[1].0 && w[1].1 == 2400));
    }

    #[test]
    fn part_timeout_covers_two_parts_at_8_kb_per_second() {
        assert_eq!(PART_TIMEOUT, Duration::from_secs(128));
    }

    #[tokio::test(start_paused = true)]
    async fn a_flood_wait_over_the_threshold_comes_back_without_sleeping() {
        let (mut session, mut sent) = Session::scripted(vec![Err(Error::Rpc(RpcError::parse(420, "FLOOD_WAIT_120")))]);
        let retry = Retry::new(AutoSleep::new().threshold(Duration::from_secs(60)));
        session.set_options(Options::new().retry(retry));
        let part = functions::upload::SaveFilePart { file_id: 1, file_part: 0, bytes: vec![0; 4] };

        let start = tokio::time::Instant::now();
        let e = invoke_part(&session, &part).await.unwrap_err();
        assert_eq!(start.elapsed(), Duration::ZERO);
        assert!(matches!(e, Error::Rpc(e) if (e.name.as_str(), e.value) == ("FLOOD_WAIT", Some(120))));
        assert_eq!(std::iter::from_fn(|| sent.try_recv().ok()).count(), 1);
    }

    #[tokio::test(start_paused = true)]
    async fn parts_follow_the_flood_sleep_setting() {
        let flood = || Err(Error::Rpc(RpcError::parse(420, "FLOOD_WAIT_1")));
        let (mut session, mut sent) = Session::scripted(vec![flood(), flood()]);
        let retry = Retry::new(AutoSleep::new().tries(1));
        session.set_options(Options::new().retry(retry));
        let part = functions::upload::SaveFilePart { file_id: 1, file_part: 0, bytes: vec![0; 4] };
        let e = invoke_part(&session, &part).await.unwrap_err();
        assert!(matches!(e, Error::Rpc(e) if e.is("FLOOD_WAIT")));
        assert_eq!(std::iter::from_fn(|| sent.try_recv().ok()).count(), 2);
    }

    type Asked = (&'static str, u32, Option<Duration>);
    struct Record(Arc<Mutex<Vec<Asked>>>);

    impl RetryPolicy for Record {
        fn should_retry(&self, ctx: &RetryContext<'_>) -> ControlFlow<(), Duration> {
            self.0.lock().unwrap().push((ctx.request, ctx.fail_count, ctx.flood_wait));
            ControlFlow::Break(())
        }
    }

    #[tokio::test(start_paused = true)]
    async fn cdn_floods_reach_the_policy_capped() {
        let seen = Arc::default();
        let flood = || Err(Error::Rpc(RpcError::parse(420, "FLOOD_WAIT_300")));
        let (mut session, _sent) = Session::scripted(vec![flood(), flood()]);
        session.set_options(Options::new().retry(Retry::new(Record(Arc::clone(&seen)))));
        let part = functions::upload::SaveFilePart { file_id: 1, file_part: 0, bytes: vec![0; 4] };
        let _ = invoke_part(&session, &part).await;
        let _ = invoke_cdn_part(&session, &part).await;
        let name = "upload.saveFilePart";
        let expected = [(name, 1, Some(Duration::from_secs(300))), (name, 1, Some(MAX_CDN_FLOOD_WAIT))];
        assert_eq!(*seen.lock().unwrap(), expected);
    }

    #[tokio::test]
    async fn close_all_fails_leased_connections_and_empties_the_pool() {
        let dc = TestDc::start([5; 256], false).await;
        let export = types::auth::ExportedAuthorization { id: 1, bytes: vec![1] };
        let export = base::auth::ExportedAuthorization::ExportedAuthorization(Box::new(export)).to_bytes();
        let (home, _sent) = Session::scripted(vec![Ok(export)]);
        let options = Options::new().transport(Transport::Intermediate);
        let pool = TransferPool::default();
        let lease = pool.lease(dc.dc(4), 1, Auth::Home(&home, 1), &options, &Arc::default()).await.unwrap();
        let ping = functions::Ping { ping_id: 1 };
        assert!(lease.sessions[0].invoke(&ping).await.is_ok());

        pool.close_all();
        assert!(matches!(lease.sessions[0].invoke(&ping).await, Err(Error::Disconnected)));
        assert!(pool.slots.lock().unwrap().is_empty());
    }

    #[tokio::test]
    async fn a_lease_opening_during_close_all_is_closed_too() {
        let client = Client::offline();
        // Nothing listens here: the lease must get by on the session it already has.
        let dc = Dc { id: 4, addr: ([127, 0, 0, 1], 1).into(), test: false };
        let (foreign, _sent) = Session::scripted(Vec::new());
        let pool = Arc::new(TransferPool::default());
        pool.insert(dc, &foreign);
        let slot = pool.slots.lock().unwrap()[&(4, false)].clone();
        let opening = slot.opening.lock().await;
        let lease = tokio::spawn({
            let (pool, home, options, events) =
                (pool.clone(), client.session(), client.0.options.clone(), client.0.events.clone());
            async move { pool.lease(dc, 1, Auth::Home(&home, 1), &options, &events).await.map(|l| l.sessions.len()) }
        });
        for _ in 0..10 {
            tokio::task::yield_now().await;
        }
        pool.close_all();
        drop(opening);
        assert!(matches!(lease.await.unwrap(), Err(Error::Disconnected)));
    }

    #[test]
    fn hands_out_each_part_once() {
        let next = AtomicI32::new(0);
        let taken: Vec<_> = std::iter::from_fn(|| take(&next, 3)).collect();
        assert_eq!(taken, [0, 1, 2]);
        assert_eq!(take(&next, 3), None);
    }

    #[tokio::test]
    async fn joins_in_order_and_stops_at_the_first_error() {
        let next = AtomicI32::new(0);
        let worker = |id: i32| {
            let next = &next;
            async move {
                let mut mine = Vec::new();
                while let Some(part) = take(next, 10) {
                    mine.push(part);
                    tokio::task::yield_now().await;
                }
                Result::Ok((id, mine))
            }
        };
        let done = try_join_all((0..3).map(worker)).await.unwrap();
        assert_eq!(done.iter().map(|d| d.0).collect::<Vec<_>>(), [0, 1, 2]);
        let mut parts: Vec<_> = done.into_iter().flat_map(|d| d.1).collect();
        parts.sort();
        assert_eq!(parts, (0..10).collect::<Vec<_>>());

        let failing = [Err(Error::Timeout), Ok(1)].map(|r| async move { r });
        assert!(matches!(try_join_all(failing.into_iter()).await, Err(Error::Timeout)));
    }
}