use std::future::Future;
use std::path::{Path, PathBuf};
use std::time::Instant;
use serde::Serialize;
use tokio::sync::mpsc;
use crate::error::{ExcelError, Result};
use crate::model::ExcelRow;
use crate::write::{ExcelWriter, SheetOptions};
#[derive(Debug, Clone)]
pub struct Page<T> {
pub rows: Vec<T>,
pub cursor: Option<String>,
pub done: bool,
}
impl<T> Page<T> {
pub fn last(rows: Vec<T>) -> Self {
Self {
rows,
cursor: None,
done: true,
}
}
pub fn more(rows: Vec<T>, cursor: impl Into<String>) -> Self {
Self {
rows,
cursor: Some(cursor.into()),
done: false,
}
}
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExportStats {
pub rows: u64,
pub pages: u64,
pub file_size: u64,
pub elapsed_ms: u64,
pub path: String,
}
#[derive(Debug, Clone)]
pub struct ExportRunner {
pub sheet: SheetOptions,
pub queue_pages: usize,
}
enum WriteMsg<T> {
Rows(Vec<T>),
EndSheet,
}
impl ExportRunner {
pub fn new(sheet: SheetOptions) -> Self {
Self {
sheet,
queue_pages: 2,
}
}
pub fn queue_pages(mut self, pages: usize) -> Self {
self.queue_pages = pages.max(1);
self
}
pub async fn export_xlsx<T, F, Fut>(
&self,
path: impl AsRef<Path>,
mut fetch: F,
) -> Result<ExportStats>
where
T: ExcelRow,
F: FnMut(Option<String>) -> Fut,
Fut: Future<Output = Result<Page<T>>> + Send,
{
let started = Instant::now();
let out_path: PathBuf = path.as_ref().to_path_buf();
let (tx, mut rx) = mpsc::channel::<Vec<T>>(self.queue_pages);
let sheet = self.sheet.clone();
let write_path = out_path.clone();
let writer_task = tokio::task::spawn_blocking(move || -> Result<u64> {
let mut writer = ExcelWriter::new();
let mut ws = writer.add_sheet(&sheet)?;
ws.write_headers()?;
while let Some(rows) = rx.blocking_recv() {
for row in &rows {
ws.write_model(row)?;
}
}
ws.finish()?;
drop(ws);
writer.save(&write_path)
});
let mut cursor: Option<String> = None;
let mut pages = 0u64;
let mut rows_total = 0u64;
let fetch_result: Result<()> = async {
loop {
let page = fetch(cursor.clone()).await?;
pages += 1;
let done = page.done;
let next = page.cursor.clone();
rows_total += page.rows.len() as u64;
if !page.rows.is_empty() {
tx.send(page.rows)
.await
.map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
}
if done {
break;
}
if next.is_none() {
log::warn!("导出分页既没有 done 也没有 cursor,提前结束");
break;
}
cursor = next;
}
Ok(())
}
.await;
drop(tx);
let file_size = writer_task
.await
.map_err(|e| ExcelError::Other(format!("导出写出线程异常:{e}")))??;
if let Err(e) = fetch_result {
let _ = std::fs::remove_file(&out_path);
return Err(e);
}
Ok(ExportStats {
rows: rows_total,
pages,
file_size,
elapsed_ms: started.elapsed().as_millis() as u64,
path: out_path.to_string_lossy().into_owned(),
})
}
pub async fn export_multi_sheet<T, F, Fut>(
&self,
path: impl AsRef<Path>,
sheets: Vec<SheetOptions>,
mut fetch: F,
) -> Result<ExportStats>
where
T: ExcelRow,
F: FnMut(usize, Option<String>) -> Fut,
Fut: Future<Output = Result<Page<T>>> + Send,
{
if sheets.is_empty() {
return Err(ExcelError::Other("导出至少需要一个工作表".to_string()));
}
let started = Instant::now();
let out_path: PathBuf = path.as_ref().to_path_buf();
let sheet_count = sheets.len();
let write_path = out_path.clone();
let (tx, mut rx) = mpsc::channel::<WriteMsg<T>>(self.queue_pages.max(1));
let writer_task = tokio::task::spawn_blocking(move || -> Result<u64> {
let mut writer = ExcelWriter::new();
for options in &sheets {
let mut ws = writer.add_sheet(options)?;
ws.write_headers()?;
while let Some(WriteMsg::Rows(rows)) = rx.blocking_recv() {
for row in &rows {
ws.write_model(row)?;
}
}
ws.finish()?;
}
writer.save(&write_path)
});
let mut pages = 0u64;
let mut rows_total = 0u64;
let fetch_result: Result<()> = async {
for index in 0..sheet_count {
let mut cursor: Option<String> = None;
loop {
let page = fetch(index, cursor.clone()).await?;
pages += 1;
let done = page.done;
let next = page.cursor.clone();
rows_total += page.rows.len() as u64;
if !page.rows.is_empty() {
tx.send(WriteMsg::Rows(page.rows))
.await
.map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
}
if done {
break;
}
if next.is_none() {
log::warn!("导出分页既没有 done 也没有 cursor,提前结束(sheet {index})");
break;
}
cursor = next;
}
tx.send(WriteMsg::EndSheet)
.await
.map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
}
Ok(())
}
.await;
drop(tx);
let file_size = writer_task
.await
.map_err(|e| ExcelError::Other(format!("导出写出线程异常:{e}")))??;
if let Err(e) = fetch_result {
let _ = std::fs::remove_file(&out_path);
return Err(e);
}
Ok(ExportStats {
rows: rows_total,
pages,
file_size,
elapsed_ms: started.elapsed().as_millis() as u64,
path: out_path.to_string_lossy().into_owned(),
})
}
pub fn export_sheets<T: ExcelRow>(
&self,
path: impl AsRef<Path>,
sheets: Vec<(SheetOptions, Vec<T>)>,
) -> Result<ExportStats> {
let started = Instant::now();
let out_path = path.as_ref().to_path_buf();
let mut rows_total = 0u64;
let mut writer = ExcelWriter::new();
for (options, rows) in &sheets {
let mut ws = writer.add_sheet(options)?;
ws.write_headers()?;
ws.write_models(rows)?;
ws.finish()?;
rows_total += rows.len() as u64;
}
let file_size = writer.save(&out_path)?;
Ok(ExportStats {
rows: rows_total,
pages: sheets.len() as u64,
file_size,
elapsed_ms: started.elapsed().as_millis() as u64,
path: out_path.to_string_lossy().into_owned(),
})
}
pub fn export_rows<T: ExcelRow>(&self, path: impl AsRef<Path>, rows: &[T]) -> Result<u64> {
let mut writer = ExcelWriter::new();
{
let mut ws = writer.add_sheet(&self.sheet)?;
ws.write_headers()?;
ws.write_models(rows)?;
ws.finish()?;
}
writer.save(path)
}
pub fn export_bytes<T: ExcelRow>(&self, rows: &[T]) -> Result<Vec<u8>> {
let mut writer = ExcelWriter::new();
{
let mut ws = writer.add_sheet(&self.sheet)?;
ws.write_headers()?;
ws.write_models(rows)?;
ws.finish()?;
}
writer.save_to_buffer()
}
}