use std::collections::HashMap;
use std::future::{Future, poll_fn};
use std::ops::ControlFlow;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::task::Poll;
use std::time::Duration;
use super::internals::data_center::Dc;
use super::pacer::{Direction, Pace, Pacer};
use super::{Events, Known, Salt, Session, Tally, WeakSession, init_connection};
use crate::client::quiet;
use crate::connection::{MIN_RATE, Options};
use crate::crypto::rsa::PublicKey;
use crate::enums::ClientEvent;
use crate::errors::{Error, Result};
use crate::raw::{RemoteCall, base, functions};
const IDLE: Duration = Duration::from_secs(15);
pub(crate) const MIN_PART_TIMEOUT: Duration = Duration::from_secs(2 * (512 << 10) / MIN_RATE);
const MAX_TIMEOUTS: usize = 3;
const MAX_SERVER_ERRORS: u32 = 6;
const FIRST_SERVER_ERROR_WAIT: Duration = Duration::from_secs(1);
const MAX_CDN_FLOOD_WAIT: Duration = Duration::from_secs(60);
#[derive(Default)]
pub(crate) struct TransferPool {
slots: Mutex<HashMap<(i32, bool), Arc<Slot>>>,
epoch: AtomicU64,
shut: AtomicBool,
pacers: Mutex<HashMap<(i32, Direction), Arc<Pacer>>>,
}
#[derive(Default)]
struct Slot {
opening: tokio::sync::Mutex<()>,
state: Mutex<State>,
}
#[derive(Default)]
struct State {
auth_key: Option<[u8; 256]>,
key_made: Option<tokio::time::Instant>,
salt: Salt,
sessions: HashMap<Dc, Vec<WeakSession>>,
opened: u32,
}
#[derive(Clone, Copy)]
pub(crate) enum Auth<'a> {
Home(&'a Session, i32),
Cdn(&'a PublicKey),
}
pub(crate) struct Lease {
pub(crate) sessions: Vec<Session>,
}
impl TransferPool {
pub(crate) fn clear(&self) {
self.slots.lock().unwrap().clear();
}
pub(crate) fn forget(&self, dc: Dc) {
self.slots.lock().unwrap().remove(&(dc.id, dc.test));
}
pub(crate) fn close_all(&self) {
let slots: Vec<Arc<Slot>> = {
let mut map = self.slots.lock().unwrap();
self.epoch.fetch_add(1, Ordering::AcqRel);
map.drain().map(|(_, slot)| slot).collect()
};
for slot in slots {
let state = slot.state.lock().unwrap();
state.sessions.values().flatten().filter_map(WeakSession::upgrade).for_each(|s| s.close());
}
}
pub(crate) fn shut(&self) {
self.shut.store(true, Ordering::Release);
self.close_all();
}
pub(crate) fn pacer(&self, dc_id: i32, direction: Direction, rate: u64) -> Arc<Pacer> {
let mut pacers = self.pacers.lock().unwrap();
pacers.entry((dc_id, direction)).or_insert_with(|| Arc::new(Pacer::new(rate))).clone()
}
#[cfg(test)]
pub(crate) fn insert(&self, dc: Dc, session: &Session) {
let slot = self.slots.lock().unwrap().entry((dc.id, dc.test)).or_default().clone();
let mut state = slot.state.lock().unwrap();
(state.auth_key, state.key_made, state.salt) = (Some(*session.auth_key()), session.key_made, session.salt());
state.sessions.entry(dc).or_default().push(session.downgrade());
}
pub(crate) async fn lease(
&self,
dc: Dc,
count: usize,
auth: Auth<'_>,
options: &Options,
events: &Arc<Events>,
) -> Result<Lease> {
self.lease_keyed_at(dc, dc, count, auth, options, events).await
}
pub(crate) async fn lease_keyed_at(
&self,
main: Dc,
dc: Dc,
count: usize,
auth: Auth<'_>,
options: &Options,
events: &Arc<Events>,
) -> Result<Lease> {
let rebind = || match auth {
Auth::Home(home, _) => home.pfs.clone(),
Auth::Cdn(_) => None,
};
if self.shut.load(Ordering::Acquire) {
return Err(Error::Disconnected);
}
let (epoch, slot) = {
let mut map = self.slots.lock().unwrap();
(self.epoch.load(Ordering::Acquire), map.entry((dc.id, dc.test)).or_default().clone())
};
let _opening = slot.opening.lock().await;
let number = || {
let mut state = slot.state.lock().unwrap();
state.opened += 1;
state.opened
};
let (mut sessions, key) = {
let state = slot.state.lock().unwrap();
let live: Vec<Session> =
state.sessions.get(&dc).into_iter().flatten().filter_map(WeakSession::upgrade).collect();
let home = match auth {
Auth::Home(home, _) if home.dc_id() == dc.id => Some(home.known_key()),
_ => None,
};
let salt = live.first().map_or(state.salt, Session::salt);
(live, home.or(state.auth_key.map(|key| Known { key, salt, made: state.key_made })))
};
let mut keyed = None;
let key = match key {
Some(key) => key,
None => {
let cdn_key = match auth {
Auth::Cdn(key) => Some(key),
Auth::Home(..) => None,
};
let session =
Session::connect_transfer(main, None, cdn_key, options, events.clone(), rebind(), number()).await?;
if let Auth::Home(home, api_id) = auth {
let export = functions::auth::ExportAuthorization { dc_id: dc.id };
let base::auth::ExportedAuthorization::ExportedAuthorization(e) = match options.no_updates {
true => home.invoke(&quiet(export)).await?,
false => home.invoke(&export).await?,
};
let import = functions::auth::ImportAuthorization { id: e.id, bytes: e.bytes };
match options.no_updates {
true => session.invoke(&init_connection(api_id, options, quiet(import))).await?,
false => session.invoke(&init_connection(api_id, options, import)).await?,
};
}
let key = session.known_key();
let mut state = slot.state.lock().unwrap();
(state.auth_key, state.key_made) = (Some(key.key), key.made);
drop(state);
match main == dc {
true => sessions.push(session),
false => keyed = Some(session),
}
key
}
};
let missing = count.saturating_sub(sessions.len());
let connect = |_| {
let opening = Session::connect_transfer(dc, Some(key), None, options, events.clone(), rebind(), number());
async move { Ok(opening.await) }
};
let mut failed = Vec::new();
for opened in try_join_all((0..missing).map(connect)).await? {
match opened {
Ok(session) => sessions.push(session),
Err(e) => failed.push(e),
}
}
let last = if sessions.is_empty() { failed.pop() } else { None };
for error in failed {
events.emit(ClientEvent::ConnectFailed { dc: dc.id, error: Arc::new(error) });
}
if let Some(e) = last {
return Err(e);
}
let mut state = slot.state.lock().unwrap();
if self.shut.load(Ordering::Acquire) || self.epoch.load(Ordering::Acquire) != epoch {
drop(state);
sessions.iter().chain(&keyed).for_each(Session::close);
return Err(Error::Disconnected);
}
state.salt = sessions.first().map_or(key.salt, Session::salt);
state.sessions.insert(dc, sessions.iter().map(Session::downgrade).collect());
if let Some(keyed) = &keyed {
state.sessions.entry(main).or_default().push(keyed.downgrade());
}
drop(state);
drop(keyed.map(|session| Lease { sessions: vec![session] }));
sessions.truncate(count);
Ok(Lease { sessions })
}
}
impl Drop for Lease {
fn drop(&mut self) {
let sessions = std::mem::take(&mut self.sessions);
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
runtime.spawn(async move {
tokio::time::sleep(IDLE).await;
drop(sessions);
});
}
}
}
pub(crate) fn part_timeout(in_flight: u64) -> Duration {
MIN_PART_TIMEOUT.max(Duration::from_secs(in_flight / MIN_RATE))
}
pub(crate) async fn invoke_part<R: RemoteCall>(session: &Session, request: &R, timeout: Duration) -> Result<R::Return> {
invoke_part_on(session, request, timeout, false, Pace::default()).await
}
pub(crate) async fn invoke_paced_part<R: RemoteCall>(
session: &Session,
request: &R,
timeout: Duration,
pace: Pace<'_>,
) -> Result<R::Return> {
invoke_part_on(session, request, timeout, false, pace).await
}
pub(crate) async fn invoke_cdn_part<R: RemoteCall>(
session: &Session,
request: &R,
timeout: Duration,
) -> Result<R::Return> {
invoke_part_on(session, request, timeout, true, Pace::default()).await
}
async fn invoke_part_on<R: RemoteCall>(
session: &Session,
request: &R,
timeout: Duration,
cdn: bool,
pace: Pace<'_>,
) -> Result<R::Return> {
let timeout = timeout.max(session.options().request_timeout);
let cap = cdn.then_some(MAX_CDN_FLOOD_WAIT);
let own = session.retries_file_parts();
let (mut timeouts, mut server_errors, mut tally) = (0, 0, Tally::default());
loop {
let gate = if pace.gated { Some(session.gate().enter().await) } else { None };
if let Some(pacer) = pace.pacer {
pacer.pace(pace.bytes).await;
}
let reconnects = session.reconnects();
let result = session.invoke_with_timeout(request, timeout).await;
let Err(e) = &result else { return result };
tally.failed(e);
let wait = match session.should_retry(R::NAME, e, tally, cap, !cdn) {
ControlFlow::Continue(wait) => wait,
ControlFlow::Break(()) if own && matches!(e, Error::Timeout) && timeouts + 1 < MAX_TIMEOUTS => {
timeouts += 1;
session.restart_since(reconnects)?;
Duration::ZERO
}
ControlFlow::Break(()) if own && !cdn && is_server_error(e) && server_errors < MAX_SERVER_ERRORS => {
server_errors += 1;
FIRST_SERVER_ERROR_WAIT * 2u32.pow(server_errors - 1)
}
ControlFlow::Break(()) => return result,
};
drop(gate);
tally.slept += wait;
tokio::time::sleep(wait).await;
}
}
fn is_server_error(e: &Error) -> bool {
matches!(e, Error::Rpc(e) if e.code >= 500 || e.code < 0 || e.is("WORKER_BUSY_TOO_LONG_RETRY"))
}
pub(crate) fn workers(parts: i32, part_size: u64, budget: u64) -> usize {
((budget / part_size) as usize).clamp(1, parts.max(1) as usize)
}
pub(crate) fn take(next: &AtomicI32, count: i32) -> Option<i32> {
let part = next.fetch_add(1, Ordering::Relaxed);
(part < count).then_some(part)
}
pub(crate) type Report<'a> = Box<dyn FnMut(u64, u64) + Send + 'a>;
pub(crate) struct Progress<'a> {
total: u64,
state: Mutex<(u64, Option<Report<'a>>)>,
}
impl<'a> Progress<'a> {
pub(crate) fn new(report: Option<Report<'a>>, total: u64) -> Self {
Progress { total, state: Mutex::new((0, report)) }
}
pub(crate) fn add(&self, bytes: usize) {
let (done, report) = &mut *self.state.lock().unwrap();
*done += bytes as u64;
if let Some(report) = report {
report(*done, self.total);
}
}
}
pub(crate) async fn try_join_all<T, F: Future<Output = Result<T>>>(futures: impl Iterator<Item = F>) -> Result<Vec<T>> {
let mut futures: Vec<_> = futures.map(|f| Some(Box::pin(f))).collect();
let mut outputs: Vec<Option<T>> = futures.iter().map(|_| None).collect();
poll_fn(|cx| {
let mut pending = false;
for (slot, output) in futures.iter_mut().zip(&mut outputs) {
let Some(future) = slot else { continue };
match future.as_mut().poll(cx) {
Poll::Ready(Ok(value)) => (*output, *slot) = (Some(value), None),
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => pending = true,
}
}
if pending { Poll::Pending } else { Poll::Ready(Ok(())) }
})
.await?;
Ok(outputs.into_iter().flatten().collect())
}
#[cfg(test)]
mod tests {
use std::ops::ControlFlow;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicI32, AtomicUsize, Ordering};
use std::time::Duration;
use super::{
Auth, MAX_CDN_FLOOD_WAIT, MIN_PART_TIMEOUT, Progress, Session, TransferPool, invoke_cdn_part,
invoke_paced_part, invoke_part, part_timeout, take, try_join_all, workers,
};
use crate::Client;
use crate::connection::slow_link::SlowLink;
use crate::connection::{MIN_RATE, Options, Transport};
use crate::enums::ClientEvent;
use crate::errors::{Error, Result, RpcError};
use crate::raw::{Serializable, base, functions, types};
use crate::session::internals::data_center::Dc;
use crate::session::pacer::{Direction, Pace, Pacer};
use crate::session::test_dc::{SAVE_BIG_FILE_PART, TestDc};
use crate::session::{AutoSleep, Known, NoRetries, Request, Retry, RetryContext, RetryPolicy, Salt};
fn part() -> functions::upload::SaveFilePart {
functions::upload::SaveFilePart { file_id: 1, file_part: 0, bytes: vec![0; 4] }
}
fn rpc_error(code: i32, message: &str) -> Result<Vec<u8>> {
Err(Error::Rpc(RpcError::parse(code, message)))
}
fn count(sent: &mut tokio::sync::mpsc::UnboundedReceiver<Vec<u8>>) -> usize {
std::iter::from_fn(|| sent.try_recv().ok()).count()
}
#[tokio::test]
async fn a_dcs_media_address_reuses_the_key_of_its_main_address() {
let client = Client::offline();
let dc = |port| Dc { id: 4, addr: ([127, 0, 0, 1], port).into(), test: false, media: false };
let (foreign, _sent) = Session::scripted(Vec::new());
let pool = TransferPool::default();
pool.insert(dc(1), &foreign);
let home = client.session();
let lease = pool.lease(dc(2), 0, Auth::Home(&home, 1), &client.0.options, &client.0.events).await;
assert!(lease.is_ok_and(|l| l.sessions.is_empty()));
}
#[test]
fn caps_parts_in_flight_by_bytes() {
let w = |parts, budget| workers(parts, 512 * 1024, budget);
assert_eq!((w(1, 4 << 20), w(3, 4 << 20), w(100, 4 << 20), w(100, 2 << 20), w(0, 2 << 20)), (1, 3, 8, 4, 1));
}
#[test]
fn reports_growing_progress_from_parallel_parts() {
let mut seen = Vec::new();
let progress = Progress::new(Some(Box::new(|done, total| seen.push((done, total)))), 2400);
std::thread::scope(|s| {
for _ in 0..8 {
s.spawn(|| (0..100).for_each(|i| progress.add(1 + i % 5)));
}
});
drop(progress);
assert_eq!((seen.len(), seen.last().copied()), (800, Some((2400, 2400))));
assert!(seen.windows(2).all(|w| w[0].0 < w[1].0 && w[1].1 == 2400));
}
#[test]
fn a_part_gets_the_time_for_every_byte_its_connection_has_asked_for() {
let least = Duration::from_secs(128);
assert_eq!([MIN_PART_TIMEOUT, part_timeout(512 << 10), part_timeout(1 << 20)], [least; 3]);
assert_eq!(part_timeout(2 << 20), Duration::from_secs(256));
}
#[tokio::test(start_paused = true)]
async fn a_flood_wait_over_the_threshold_comes_back_without_sleeping() {
let (mut session, mut sent) = Session::scripted(vec![rpc_error(420, "FLOOD_WAIT_120")]);
let retry = Retry::new(AutoSleep::new().threshold(Duration::from_secs(60)));
session.set_options(Options::new().retry(retry));
let start = tokio::time::Instant::now();
let e = invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap_err();
assert_eq!(start.elapsed(), Duration::ZERO);
assert!(matches!(e, Error::Rpc(e) if (e.name.as_str(), e.value) == ("FLOOD_WAIT", Some(120))));
assert_eq!(count(&mut sent), 1);
}
#[tokio::test(start_paused = true)]
async fn parts_follow_the_flood_sleep_setting_four_times_over() {
let (mut session, mut sent) = Session::scripted((0..5).map(|_| rpc_error(420, "FLOOD_WAIT_1")).collect());
session.set_options(Options::new().retry(Retry::new(AutoSleep::new().tries(1))));
let e = invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap_err();
assert!(matches!(e, Error::Rpc(e) if e.is("FLOOD_WAIT")));
assert_eq!(count(&mut sent), 5);
}
#[tokio::test(start_paused = true)]
async fn a_part_outlasts_the_floods_that_would_fail_a_request() {
let mut replies: Vec<_> = (0..6).map(|_| rpc_error(420, "FLOOD_WAIT_2")).collect();
replies.push(Ok(true.to_bytes()));
let (session, mut sent) = Session::scripted(replies);
assert!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap());
assert_eq!(count(&mut sent), 7);
}
#[tokio::test(start_paused = true)]
async fn server_errors_are_waited_out_a_growing_while() {
let replies = vec![
rpc_error(500, "INTERNAL"),
rpc_error(-503, "Timeout"),
rpc_error(500, "WORKER_BUSY_TOO_LONG_RETRY"),
Ok(true.to_bytes()),
];
let (session, mut sent) = Session::scripted(replies);
let start = tokio::time::Instant::now();
assert!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap());
assert_eq!((start.elapsed(), count(&mut sent)), (Duration::from_secs(1 + 2 + 4), 4));
}
#[tokio::test]
async fn transfers_to_one_dc_share_a_pacer_and_other_dcs_get_their_own() {
let pool = TransferPool::default();
let first = pool.pacer(2, Direction::Download, 100);
assert!(Arc::ptr_eq(&first, &pool.pacer(2, Direction::Download, 100)));
assert!(!Arc::ptr_eq(&first, &pool.pacer(4, Direction::Download, 100)));
assert!(!Arc::ptr_eq(&first, &pool.pacer(2, Direction::Upload, 100)));
}
async fn seconds(
sent: &mut tokio::sync::mpsc::UnboundedReceiver<Vec<u8>>,
start: tokio::time::Instant,
n: usize,
) -> Vec<u64> {
let mut at = Vec::new();
while at.len() < n && sent.recv().await.is_some() {
at.push(start.elapsed().as_secs());
}
at
}
#[tokio::test(start_paused = true)]
async fn a_gated_part_sleeping_off_a_flood_lets_another_have_its_connection() {
let ok = || Ok(true.to_bytes());
let (session, mut sent) = Session::scripted(vec![rpc_error(420, "FLOOD_WAIT_3"), ok(), ok()]);
let start = tokio::time::Instant::now();
let pace = Pace { pacer: None, bytes: 0, gated: true };
let send = |file_part| {
let session = &session;
async move {
let part = functions::upload::SaveFilePart { file_id: 1, file_part, bytes: vec![0; 4] };
invoke_paced_part(session, &part, MIN_PART_TIMEOUT, pace).await
}
};
let (saved, sent_at) = tokio::join!(try_join_all((0..2).map(send)), seconds(&mut sent, start, 3));
assert_eq!(saved.unwrap(), [true; 2]);
assert_eq!(sent_at, [0, 0, 3]);
}
#[tokio::test(start_paused = true)]
async fn a_flood_on_one_paced_part_holds_back_only_that_part() {
const PART: u64 = 512 << 10;
let ok = || Ok(true.to_bytes());
let (session, mut sent) = Session::scripted(vec![rpc_error(420, "FLOOD_WAIT_3"), ok(), ok(), ok(), ok()]);
let pacer = Pacer::new(PART);
let start = tokio::time::Instant::now();
let pace = Pace { pacer: Some(&pacer), bytes: PART, gated: false };
let send = |file_part| {
let session = &session;
async move {
let part = functions::upload::SaveFilePart { file_id: 1, file_part, bytes: vec![0; PART as usize] };
invoke_paced_part(session, &part, MIN_PART_TIMEOUT, pace).await
}
};
let sent_at = seconds(&mut sent, start, 5);
let (saved, sent_at) = tokio::join!(try_join_all((0..4).map(send)), sent_at);
assert_eq!(saved.unwrap(), [true; 4]);
assert_eq!(sent_at, [0, 1, 2, 3, 4]);
}
#[tokio::test(start_paused = true)]
async fn server_errors_are_given_up_on_after_a_minute() {
let (session, mut sent) = Session::scripted((0..7).map(|_| rpc_error(500, "INTERNAL")).collect());
let start = tokio::time::Instant::now();
let e = invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap_err();
assert!(matches!(e, Error::Rpc(e) if e.code == 500));
assert_eq!((start.elapsed(), count(&mut sent)), (Duration::from_secs(1 + 2 + 4 + 8 + 16 + 32), 7));
}
#[tokio::test(start_paused = true)]
async fn a_cdn_part_hands_a_server_error_over_at_once() {
let (session, mut sent) = Session::scripted(vec![rpc_error(500, "INTERNAL")]);
assert!(matches!(invoke_cdn_part(&session, &part(), MIN_PART_TIMEOUT).await, Err(Error::Rpc(_))));
assert_eq!(count(&mut sent), 1);
}
#[tokio::test(start_paused = true)]
async fn a_part_that_times_out_restarts_its_connection_and_goes_again() {
let (requests, mut rx) = tokio::sync::mpsc::unbounded_channel::<Request>();
tokio::spawn(async move {
let Some(lost) = rx.recv().await else { return };
while let Some(request) = rx.recv().await {
let _ = request.answer.send(Ok(true.to_bytes()));
}
drop(lost);
});
let session = Session::local(requests);
let start = tokio::time::Instant::now();
assert!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap());
assert!((MIN_PART_TIMEOUT..MIN_PART_TIMEOUT + Duration::from_secs(1)).contains(&start.elapsed()));
assert!(session.control.take_restart(), "the part went again on the same connection");
}
type Asked = (&'static str, u32, Option<Duration>, bool);
struct Record(Arc<Mutex<Vec<Asked>>>);
impl RetryPolicy for Record {
fn should_retry(&self, ctx: &RetryContext<'_>) -> ControlFlow<(), Duration> {
self.0.lock().unwrap().push((ctx.request, ctx.fail_count, ctx.flood_wait, ctx.file_part));
ControlFlow::Break(())
}
}
#[tokio::test(start_paused = true)]
async fn cdn_floods_reach_the_policy_capped_and_not_as_file_parts() {
let seen = Arc::default();
let flood = || rpc_error(420, "FLOOD_WAIT_300");
let (mut session, _sent) = Session::scripted(vec![flood(), flood()]);
session.set_options(Options::new().retry(Retry::new(Record(Arc::clone(&seen)))));
let _ = invoke_part(&session, &part(), MIN_PART_TIMEOUT).await;
let _ = invoke_cdn_part(&session, &part(), MIN_PART_TIMEOUT).await;
let name = "upload.saveFilePart";
let expected = [(name, 1, Some(Duration::from_secs(300)), true), (name, 1, Some(MAX_CDN_FLOOD_WAIT), false)];
assert_eq!(*seen.lock().unwrap(), expected);
}
#[tokio::test(start_paused = true)]
async fn the_policy_hears_of_every_failure_of_a_part() {
let seen = Arc::default();
let replies = vec![rpc_error(500, "INTERNAL"), rpc_error(500, "INTERNAL"), Ok(true.to_bytes())];
let (mut session, _sent) = Session::scripted(replies);
session.set_options(Options::new().retry(Retry::new(Record(Arc::clone(&seen)))));
assert!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap());
let asked: Vec<u32> = seen.lock().unwrap().iter().map(|a| a.1).collect();
assert_eq!(asked, [1, 2]);
}
#[tokio::test(start_paused = true)]
async fn no_retries_sends_a_part_once_whatever_it_fails_with() {
let (mut session, mut sent) = Session::scripted(vec![rpc_error(500, "INTERNAL")]);
session.set_options(Options::new().retry(Retry::new(NoRetries)));
assert!(matches!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await, Err(Error::Rpc(_))));
assert_eq!(count(&mut sent), 1);
let (requests, mut rx) = tokio::sync::mpsc::unbounded_channel::<Request>();
let held = tokio::spawn(async move {
let mut held = Vec::new();
while let Some(request) = rx.recv().await {
held.push(request);
}
held.len()
});
let mut session = Session::local(requests);
session.set_options(Options::new().retry(Retry::new(NoRetries)));
assert!(matches!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await, Err(Error::Timeout)));
assert!(!session.control.take_restart());
drop(session);
assert_eq!(held.await.unwrap(), 1);
}
#[tokio::test]
async fn parts_that_timed_out_together_restart_their_connection_once() {
let (requests, _rx) = tokio::sync::mpsc::unbounded_channel::<Request>();
let session = Session::local(requests);
let control = &session.control;
let sent_at = session.reconnects();
session.restart_since(sent_at).unwrap();
session.restart_since(sent_at).unwrap();
assert!(control.take_restart());
session.restart_since(sent_at).unwrap();
control.dropped();
control.take_restart();
session.restart_since(sent_at).unwrap();
assert!(!control.take_restart(), "a second reconnect for the same dead connection");
session.restart_since(session.reconnects()).unwrap();
assert!(control.take_restart());
}
#[tokio::test(start_paused = true)]
async fn a_part_on_a_slow_uplink_waits_for_its_answer() {
let dc = TestDc::start([5; 256], false).await;
let serve = dc.acceptor();
let factory = move |_: std::net::SocketAddr| {
let (link, far) = SlowLink::new(1 << 20, 1, MIN_RATE as usize);
serve(far);
async move { Ok::<_, std::io::Error>(link) }
};
let options = Options::new().transport(Transport::Intermediate).connection_factory(factory);
let key = Known { key: [5; 256], salt: Salt::default(), made: None };
let session =
Session::connect_transfer(dc.dc(2), Some(key), None, &options, Arc::default(), None, 1).await.unwrap();
let bytes = vec![0; 512 << 10];
let part = functions::upload::SaveBigFilePart { file_id: 1, file_part: 0, file_total_parts: 1, bytes };
let start = tokio::time::Instant::now();
assert!(invoke_part(&session, &part, part_timeout(512 << 10)).await.unwrap());
let took = start.elapsed();
assert!((Duration::from_secs(64)..Duration::from_secs(70)).contains(&took), "{took:?}");
let parts = dc.seen().into_iter().filter(|s| s.ids.contains(&SAVE_BIG_FILE_PART)).map(|s| s.connection);
assert_eq!(parts.collect::<Vec<_>>(), [1]);
}
type Opened = Arc<Mutex<Vec<(tokio::time::Instant, Arc<AtomicUsize>)>>>;
async fn linked(dc: &TestDc, room: usize, rate: usize) -> (Session, Opened) {
let opened = Opened::default();
let (serve, log) = (dc.acceptor(), Arc::clone(&opened));
let factory = move |_: std::net::SocketAddr| {
let (link, far) = SlowLink::new(room, 1, rate);
log.lock().unwrap().push((tokio::time::Instant::now(), link.rate()));
serve(far);
async move { Ok::<_, std::io::Error>(link) }
};
let options = Options::new().transport(Transport::Intermediate).connection_factory(factory);
let key = Known { key: [5; 256], salt: Salt::default(), made: None };
let session =
Session::connect_transfer(dc.dc(2), Some(key), None, &options, Arc::default(), None, 1).await.unwrap();
(session, opened)
}
fn big_part(file_part: i32, len: usize) -> functions::upload::SaveBigFilePart {
functions::upload::SaveBigFilePart { file_id: 1, file_part, file_total_parts: 201, bytes: vec![0; len] }
}
async fn answered_parts(session: &Session) {
for round in 0..10 {
let parts: Vec<_> = (0..20).map(|i| big_part(round * 20 + i, 32 << 10)).collect();
let done = try_join_all(parts.iter().map(|p| invoke_part(session, p, MIN_PART_TIMEOUT))).await.unwrap();
assert!(done.into_iter().all(|saved| saved));
}
}
#[tokio::test(start_paused = true)]
async fn a_connection_gone_quiet_after_many_parts_is_dropped_within_its_last_parts_time() {
let dc = TestDc::start([5; 256], false).await;
let (session, opened) = linked(&dc, 1 << 20, 1 << 20).await;
let start = opened.lock().unwrap()[0].0;
answered_parts(&session).await;
assert!(start.elapsed() < Duration::from_secs(15), "{:?}", start.elapsed());
tokio::time::sleep_until(start + Duration::from_secs(16)).await;
dc.mute();
let sent = tokio::time::Instant::now();
let last = big_part(200, 32 << 10);
let _ = tokio::time::timeout(Duration::from_secs(60), invoke_part(&session, &last, MIN_PART_TIMEOUT)).await;
let opened: Vec<_> = opened.lock().unwrap().iter().map(|o| o.0).collect();
assert!(opened.len() > 1, "a connection that went quiet was never dropped");
let dropped = opened[1] - sent;
assert!((Duration::from_secs(36)..Duration::from_secs(37)).contains(&dropped), "{dropped:?}");
}
#[tokio::test(start_paused = true)]
async fn a_peer_that_stops_taking_bytes_after_many_parts_fails_the_write_within_its_buffers_time() {
let dc = TestDc::start([5; 256], false).await;
let (session, opened) = linked(&dc, 256 << 10, 1 << 20).await;
answered_parts(&session).await;
opened.lock().unwrap()[0].1.store(0, Ordering::Relaxed);
let sent = tokio::time::Instant::now();
let last = big_part(200, 512 << 10);
let saving = invoke_part(&session, &last, part_timeout(512 << 10));
let _ = tokio::time::timeout(Duration::from_secs(70), saving).await;
let opened: Vec<_> = opened.lock().unwrap().iter().map(|o| o.0).collect();
assert!(opened.len() > 1, "a write the peer took nothing of never failed");
let failed = opened[1] - sent;
assert!((Duration::from_secs(51)..Duration::from_secs(54)).contains(&failed), "{failed:?}");
}
#[tokio::test(start_paused = true)]
async fn a_part_with_server_errors_behind_it_still_sleeps_off_every_flood_wait() {
let mut replies = vec![rpc_error(500, "INTERNAL"), rpc_error(500, "INTERNAL")];
replies.extend((0..20).map(|_| rpc_error(420, "FLOOD_WAIT_1")));
replies.push(Ok(true.to_bytes()));
let (session, mut sent) = Session::scripted(replies);
assert!(invoke_part(&session, &part(), MIN_PART_TIMEOUT).await.unwrap());
assert_eq!(count(&mut sent), 23);
}
#[tokio::test]
async fn a_lease_goes_on_with_the_connections_that_opened() {
let client = Client::offline();
let dc = Dc { id: 4, addr: ([127, 0, 0, 1], 1).into(), test: false, media: false };
let (open, _sent) = Session::scripted(Vec::new());
let pool = TransferPool::default();
pool.insert(dc, &open);
let home = client.session();
let events = client.0.events.record();
let lease = pool.lease(dc, 3, Auth::Home(&home, 1), &client.0.options, &client.0.events).await.unwrap();
assert_eq!(lease.sessions.len(), 1);
let failed = events.try_iter().filter(|e| matches!(e, ClientEvent::ConnectFailed { dc: 4, .. })).count();
assert_eq!(failed, 2);
}
#[tokio::test]
async fn a_key_exchange_on_a_media_address_negates_the_dc() {
let dc = TestDc::start([5; 256], false).await;
let options = Options::new().transport(Transport::Intermediate);
for (n, media) in [(1, dc.dc(4).media(true)), (2, dc.dc(4).test(true).media(true))] {
Session::connect_transfer(media, None, None, &options, Arc::default(), None, n).await.unwrap();
}
assert_eq!(dc.handshakes(), [(1, -4, false), (2, -10004, false)]);
}
#[tokio::test]
async fn a_media_lease_makes_the_key_on_the_main_address_and_temporary_keys_on_its_own() {
use crate::session::test_dc::IMPORT_AUTHORIZATION;
let dc = TestDc::start([5; 256], false).await;
let export = types::auth::ExportedAuthorization { id: 1, bytes: vec![1] };
let export = base::auth::ExportedAuthorization::ExportedAuthorization(Box::new(export)).to_bytes();
let (home, _sent) = Session::scripted(vec![Ok(export)]);
let init = crate::session::init_connection(1, &Options::default(), functions::help::GetNearestDc {});
let home = Session { pfs: Some(Arc::from(init.to_bytes())), ..home };
let options = Options::new().transport(Transport::Intermediate);
let pool = TransferPool::default();
let (main, media) = (dc.dc(4), dc.dc(4).media(true));
let lease = pool.lease_keyed_at(main, media, 2, Auth::Home(&home, 1), &options, &Arc::default()).await.unwrap();
for session in &lease.sessions {
session.invoke(&functions::Ping { ping_id: 1 }).await.unwrap();
}
let mut handshakes = dc.handshakes();
handshakes.sort();
assert_eq!(handshakes, [(1, 4, false), (1, 4, true), (2, -4, true), (3, -4, true)]);
let imports = dc.seen().into_iter().filter(|s| s.ids.contains(&IMPORT_AUTHORIZATION)).map(|s| s.connection);
assert_eq!(imports.collect::<Vec<_>>(), [1]);
assert_eq!(lease.sessions.len(), 2);
}
#[tokio::test]
async fn close_all_fails_leased_connections_and_empties_the_pool() {
let dc = TestDc::start([5; 256], false).await;
let export = types::auth::ExportedAuthorization { id: 1, bytes: vec![1] };
let export = base::auth::ExportedAuthorization::ExportedAuthorization(Box::new(export)).to_bytes();
let (home, _sent) = Session::scripted(vec![Ok(export)]);
let options = Options::new().transport(Transport::Intermediate);
let pool = TransferPool::default();
let lease = pool.lease(dc.dc(4), 1, Auth::Home(&home, 1), &options, &Arc::default()).await.unwrap();
let ping = functions::Ping { ping_id: 1 };
assert!(lease.sessions[0].invoke(&ping).await.is_ok());
pool.close_all();
assert!(matches!(lease.sessions[0].invoke(&ping).await, Err(Error::Disconnected)));
assert!(pool.slots.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_lease_opening_during_close_all_is_closed_too() {
let client = Client::offline();
let dc = Dc { id: 4, addr: ([127, 0, 0, 1], 1).into(), test: false, media: false };
let (foreign, _sent) = Session::scripted(Vec::new());
let pool = Arc::new(TransferPool::default());
pool.insert(dc, &foreign);
let slot = pool.slots.lock().unwrap()[&(4, false)].clone();
let opening = slot.opening.lock().await;
let lease = tokio::spawn({
let (pool, home, options, events) =
(pool.clone(), client.session(), client.0.options.clone(), client.0.events.clone());
async move { pool.lease(dc, 1, Auth::Home(&home, 1), &options, &events).await.map(|l| l.sessions.len()) }
});
for _ in 0..10 {
tokio::task::yield_now().await;
}
pool.close_all();
drop(opening);
assert!(matches!(lease.await.unwrap(), Err(Error::Disconnected)));
}
#[test]
fn hands_out_each_part_once() {
let next = AtomicI32::new(0);
let taken: Vec<_> = std::iter::from_fn(|| take(&next, 3)).collect();
assert_eq!(taken, [0, 1, 2]);
assert_eq!(take(&next, 3), None);
}
#[tokio::test]
async fn joins_in_order_and_stops_at_the_first_error() {
let next = AtomicI32::new(0);
let worker = |id: i32| {
let next = &next;
async move {
let mut mine = Vec::new();
while let Some(part) = take(next, 10) {
mine.push(part);
tokio::task::yield_now().await;
}
Result::Ok((id, mine))
}
};
let done = try_join_all((0..3).map(worker)).await.unwrap();
assert_eq!(done.iter().map(|d| d.0).collect::<Vec<_>>(), [0, 1, 2]);
let mut parts: Vec<_> = done.into_iter().flat_map(|d| d.1).collect();
parts.sort();
assert_eq!(parts, (0..10).collect::<Vec<_>>());
let failing = [Err(Error::Timeout), Ok(1)].map(|r| async move { r });
assert!(matches!(try_join_all(failing.into_iter()).await, Err(Error::Timeout)));
}
}