use std::pin::Pin;
use std::sync::atomic::AtomicI32;
use tokio::io::{AsyncRead, AsyncReadExt};
use crate::Client;
use crate::crypto::random;
use crate::errors::{Error, Result};
use crate::raw::{base, functions, types};
use crate::session::Session;
use crate::session::transfer::{Progress, Report, invoke_part, take, try_join_all, workers};
use crate::types::Upload;
const PART_SIZE: u64 = 512 * 1024;
const BIG_FILE_SIZE: u64 = 10 * 1024 * 1024;
const MAX_PARTS: u64 = 4000;
const MAX_PARTS_PREMIUM: u64 = 8000;
const IN_FLIGHT: u64 = 4 << 20;
const CONNECTIONS: usize = 4;
const CONNECTIONS_PREMIUM: usize = 8;
#[must_use = "a request does nothing until it is awaited"]
pub(crate) struct SaveFile<'a> {
client: &'a Client,
input: Input<'a>,
progress: Option<Report<'a>>,
}
enum Input<'a> {
Upload(Upload<'a>),
Stream { reader: Pin<Box<dyn AsyncRead + Send + 'a>>, size: u64, name: &'a str },
}
impl Client {
pub(crate) fn save_file<'a>(&'a self, file: impl Into<Upload<'a>>) -> SaveFile<'a> {
SaveFile { client: self, input: Input::Upload(file.into()), progress: None }
}
pub(crate) fn save_stream<'a>(
&'a self,
stream: impl AsyncRead + Send + 'a,
size: u64,
name: &'a str,
) -> SaveFile<'a> {
let input = Input::Stream { reader: Box::pin(stream), size, name };
SaveFile { client: self, input, progress: None }
}
}
impl<'a> SaveFile<'a> {
pub(crate) fn progress(mut self, progress: Option<Report<'a>>) -> Self {
self.progress = progress;
self
}
async fn send(self) -> Result<base::InputFile> {
let (source, size, name) = match self.input {
Input::Upload(upload @ Upload::Path(path)) => {
let disk = tokio::fs::File::open(path).await?;
let size = disk.metadata().await?.len();
(Source::File(tokio::sync::Mutex::new(disk)), size, upload.name())
}
Input::Upload(upload @ Upload::Bytes { data, .. }) => {
(Source::Bytes(data), data.len() as u64, upload.name())
}
Input::Stream { reader, size, name } => (Source::Stream(tokio::sync::Mutex::new(reader)), size, name),
};
if size == 0 {
return Err(Error::FileSize { size: 0, limit: MAX_PARTS * PART_SIZE });
}
let _slot = self.client.upload_slot().await;
let (parts, workers, connections) = self.client.upload_limits(size).await?;
let home = self.client.dc(self.client.session().dc_id())?;
let lease = self.client.transfer_sessions(home, connections).await?;
let upload = Parts { source, size, count: parts, big: size > BIG_FILE_SIZE, id: i64::from_le_bytes(random()) };
let (next, progress) = (AtomicI32::new(0), Progress::new(self.progress, size));
let sessions = &lease.sessions;
try_join_all((0..workers).map(|i| upload.send(&sessions[i % sessions.len()], &next, &progress))).await?;
Ok(upload.input_file(name))
}
}
into_future!(SaveFile, base::InputFile);
impl Client {
async fn upload_limits(&self, size: u64) -> Result<(i32, usize, usize)> {
let premium = match self.is_premium().await {
Err(e) if size > MAX_PARTS * PART_SIZE => return Err(e),
premium => premium.unwrap_or(false),
};
let parts = part_count(size, premium)?;
let workers = workers(parts, PART_SIZE, IN_FLIGHT);
Ok((parts, workers, workers.min(if premium { CONNECTIONS_PREMIUM } else { CONNECTIONS })))
}
}
fn part_count(size: u64, premium: bool) -> Result<i32> {
let limit = if premium { MAX_PARTS_PREMIUM } else { MAX_PARTS } * PART_SIZE;
if size == 0 || size > limit {
return Err(Error::FileSize { size, limit });
}
Ok(size.div_ceil(PART_SIZE) as i32)
}
enum Source<'a> {
File(tokio::sync::Mutex<tokio::fs::File>),
Stream(tokio::sync::Mutex<Pin<Box<dyn AsyncRead + Send + 'a>>>),
Bytes(&'a [u8]),
}
struct Parts<'a> {
source: Source<'a>,
size: u64,
count: i32,
big: bool,
id: i64,
}
impl Parts<'_> {
fn len(&self, part: i32) -> usize {
(self.size - part as u64 * PART_SIZE).min(PART_SIZE) as usize
}
async fn read(&self, next: &AtomicI32, buf: &mut Vec<u8>) -> Result<Option<i32>> {
match &self.source {
Source::File(file) => self.read_locked(file, next, buf).await,
Source::Stream(stream) => self.read_locked(stream, next, buf).await,
Source::Bytes(data) => {
let Some(part) = take(next, self.count) else { return Ok(None) };
let start = part as usize * PART_SIZE as usize;
buf.clear();
buf.extend_from_slice(&data[start..start + self.len(part)]);
Ok(Some(part))
}
}
}
async fn read_locked<R: AsyncRead + Unpin + Send>(
&self,
reader: &tokio::sync::Mutex<R>,
next: &AtomicI32,
buf: &mut Vec<u8>,
) -> Result<Option<i32>> {
let mut reader = reader.lock().await;
let Some(part) = take(next, self.count) else { return Ok(None) };
buf.resize(self.len(part), 0);
reader.read_exact(buf).await?;
Ok(Some(part))
}
async fn send(&self, session: &Session, next: &AtomicI32, progress: &Progress<'_>) -> Result<()> {
let (file_id, mut bytes) = (self.id, Vec::with_capacity(PART_SIZE.min(self.size) as usize));
while let Some(file_part) = self.read(next, &mut bytes).await? {
let saved = if self.big {
let total = self.count;
let request = functions::upload::SaveBigFilePart { file_id, file_part, file_total_parts: total, bytes };
let saved = invoke_part(session, &request).await;
bytes = request.bytes;
saved
} else {
let request = functions::upload::SaveFilePart { file_id, file_part, bytes };
let saved = invoke_part(session, &request).await;
bytes = request.bytes;
saved
};
if !saved? {
return Err(Error::Unexpected("upload.saveFilePart returned false"));
}
progress.add(bytes.len());
}
Ok(())
}
fn input_file(&self, name: &str) -> base::InputFile {
let (id, parts, name) = (self.id, self.count, name.to_owned());
if self.big {
base::InputFile::InputFileBig(Box::new(types::InputFileBig { id, parts, name }))
} else {
base::InputFile::InputFile(Box::new(types::InputFile { id, parts, name, md5_checksum: String::new() }))
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::AtomicI32;
use super::{BIG_FILE_SIZE, MAX_PARTS, MAX_PARTS_PREMIUM, PART_SIZE, Parts, Source, part_count};
use crate::Client;
use crate::client::Transmissions;
use crate::errors::{Error, RpcError};
use crate::raw::functions::users::GetUsers;
use crate::raw::{Identifiable, Serializable, base, types};
use crate::types::Upload;
fn premium_user() -> Vec<u8> {
let mut user = Vec::new();
for word in [1u32 << 28, 0] {
word.serialize(&mut user);
}
7i64.serialize(&mut user);
let user = types::User::deserialize_bare(&mut &user[..]).unwrap();
let mut reply = Vec::new();
vec![base::User::User(Box::new(user))].serialize(&mut reply);
reply
}
#[tokio::test]
async fn uploads_with_regular_limits_when_the_account_lookup_fails() {
let failed = Err(Error::Rpc(RpcError::parse(500, "INTERNAL")));
let (client, mut sent) = Client::scripted(vec![failed, Ok(premium_user())]);
client.0.me.lock().unwrap().is_bot = false;
let get_users = GetUsers::CONSTRUCTOR_ID.to_le_bytes();
let mut lookups = || std::iter::from_fn(|| sent.try_recv().ok()).filter(|b| b[..4] == get_users).count();
let big = MAX_PARTS * PART_SIZE + 1;
assert_eq!(client.upload_limits(3 << 20).await.unwrap(), (6, 6, 4));
assert!(matches!(client.upload_limits(big).await, Ok((4001, 8, 8))));
assert_eq!(lookups(), 2);
assert_eq!(client.upload_limits(3 << 20).await.unwrap(), (6, 6, 6));
assert_eq!(lookups(), 0);
client.0.events.reconnected();
assert!(matches!(client.upload_limits(big).await, Err(Error::Disconnected)));
assert_eq!(lookups(), 1);
}
#[test]
fn counts_512k_parts_up_to_the_account_limit() {
assert!(matches!(part_count(0, false), Err(Error::FileSize { size: 0, .. })));
let count = |size| part_count(size, false).unwrap();
assert_eq!(
(count(1), count(PART_SIZE), count(PART_SIZE + 1), count(BIG_FILE_SIZE), count(BIG_FILE_SIZE + 1)),
(1, 1, 2, 20, 21)
);
assert_eq!(count(MAX_PARTS * PART_SIZE), 4000);
assert!(matches!(part_count(MAX_PARTS * PART_SIZE + 1, false), Err(Error::FileSize { .. })));
assert_eq!(part_count(MAX_PARTS * PART_SIZE + 1, true).unwrap(), 4001);
assert!(matches!(part_count(MAX_PARTS_PREMIUM * PART_SIZE + 1, true), Err(Error::FileSize { .. })));
}
fn stream(data: &[u8]) -> Source<'_> {
Source::Stream(tokio::sync::Mutex::new(Box::pin(data)))
}
async fn split(data: &[u8]) -> Vec<usize> {
split_from(Source::Bytes(data), data).await
}
async fn split_from(source: Source<'_>, data: &[u8]) -> Vec<usize> {
let count = part_count(data.len() as u64, false).unwrap();
let big = data.len() as u64 > BIG_FILE_SIZE;
let parts = Parts { source, size: data.len() as u64, count, big, id: 7 };
let (next, mut sizes, mut joined, mut buf) = (AtomicI32::new(0), Vec::new(), Vec::new(), Vec::new());
while let Some(n) = parts.read(&next, &mut buf).await.unwrap() {
assert_eq!(n as usize, sizes.len());
sizes.push(buf.len());
joined.extend_from_slice(&buf);
}
assert_eq!(joined, data);
sizes
}
#[tokio::test]
async fn reads_parts_in_order() {
let part = PART_SIZE as usize;
assert_eq!(split(b"a").await, [1]);
assert_eq!(split(&vec![1; part]).await, [part]);
assert_eq!(split(&vec![1; part + 1]).await, [part, 1]);
let big = split(&vec![2; BIG_FILE_SIZE as usize + 1]).await;
assert_eq!((big.len(), big[20]), (21, 1));
}
#[tokio::test]
async fn reads_a_stream_in_part_order() {
let part = PART_SIZE as usize;
let data: Vec<u8> = (0..part + 1).map(|i| i as u8).collect();
assert_eq!(split_from(stream(&data[..1]), &data[..1]).await, [1]);
assert_eq!(split_from(stream(&data[..part]), &data[..part]).await, [part]);
assert_eq!(split_from(stream(&data), &data).await, [part, 1]);
}
#[tokio::test]
async fn a_short_stream_fails_with_unexpected_eof() {
let parts = Parts { source: stream(b"hello"), size: 10, count: 1, big: false, id: 7 };
let e = parts.read(&AtomicI32::new(0), &mut Vec::new()).await.err();
assert!(matches!(&e, Some(Error::Io(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof), "{e:?}");
}
#[tokio::test]
async fn an_empty_upload_is_refused_before_any_request() {
let (client, mut sent) = Client::recording();
{
let mut me = client.0.me.lock().unwrap();
(me.is_bot, me.premium) = (false, None);
}
let empty = [client.save_file(Upload::bytes("a.txt", b"")).await, client.save_stream(&b""[..], 0, "a").await];
for e in empty {
assert!(matches!(e, Err(Error::FileSize { size: 0, .. })), "{e:?}");
}
assert!(sent.try_recv().is_err());
}
#[tokio::test]
async fn an_upload_waits_for_a_free_slot_but_an_empty_one_does_not() {
let (mut client, mut sent) = Client::recording();
Arc::get_mut(&mut client.0).unwrap().transmissions = Some(Transmissions::new(1));
{
let mut me = client.0.me.lock().unwrap();
(me.is_bot, me.premium) = (false, None);
}
let held = client.upload_slot().await;
let waiting = tokio::spawn({
let client = client.clone();
async move { client.save_file(Upload::bytes("a.txt", b"x")).await.map(drop) }
});
for _ in 0..50 {
tokio::task::yield_now().await;
}
assert!(!waiting.is_finished() && sent.try_recv().is_err(), "the upload ran without a slot");
let empty = client.save_file(Upload::bytes("a.txt", b"")).await;
assert!(matches!(empty, Err(Error::FileSize { size: 0, .. })), "{empty:?}");
drop(held);
for _ in 0..50 {
tokio::task::yield_now().await;
}
assert!(sent.try_recv().is_ok(), "the upload didn't start once the slot was free");
waiting.abort();
}
#[test]
fn switches_to_big_files_above_10_mb() {
let file = |size: u64| {
let big = size > BIG_FILE_SIZE;
let count = part_count(size, false).unwrap();
Parts { source: Source::Bytes(&[]), size, count, big, id: 7 }.input_file("a.bin")
};
let base::InputFile::InputFile(small) = file(BIG_FILE_SIZE) else { panic!("10 MB is still a small file") };
assert_eq!((small.id, small.parts, small.name.as_str(), small.md5_checksum.as_str()), (7, 20, "a.bin", ""));
assert!(matches!(file(BIG_FILE_SIZE + 1), base::InputFile::InputFileBig(f) if f.parts == 21));
}
}