Skip to main content

fast_excel/
export.rs

1//! 导出侧:游标分批拉取 + 流式写出(百万行不进内存)
2//!
3//! 用法(sea-orm 游标分页为例):
4//!
5//! ```ignore
6//! let runner = ExportRunner::new(SheetOptions::new("选品数据").columns(ProductRow::columns()));
7//! let stats = runner
8//!     .export_xlsx("uploads/export/products.xlsx", |cursor| async move {
9//!         let mut q = product::Entity::find().order_by_asc(product::Column::Id).limit(1000);
10//!         if let Some(last_id) = cursor {
11//!             q = q.filter(product::Column::Id.gt(last_id));
12//!         }
13//!         let rows = q.all(&db).await?;
14//!         let done = rows.len() < 1000;
15//!         let last_id = rows.last().map(|r| r.id.clone());
16//!         Ok(Page { rows: rows.into_iter().map(Into::into).collect(), cursor: last_id, done })
17//!     })
18//!     .await?;
19//! ```
20//!
21//! 多工作表导出用 [`ExportRunner::export_multi_sheet`]:每个 sheet 各自游标分页,
22//! 依次写出,单表百万行依旧是常量内存。
23
24use std::future::Future;
25use std::path::{Path, PathBuf};
26use std::time::Instant;
27
28use serde::Serialize;
29use tokio::sync::mpsc;
30
31use crate::error::{ExcelError, Result};
32use crate::model::ExcelRow;
33use crate::write::{ExcelWriter, SheetOptions};
34
35/// 一页导出数据(游标由业务自定义:上一页最后一条 id / 时间戳 / 主键组合都可以)
36#[derive(Debug, Clone)]
37pub struct Page<T> {
38    pub rows: Vec<T>,
39    /// 下一页游标;`None` 表示没有更多(配合 `done` 使用)
40    pub cursor: Option<String>,
41    /// 是否已是最后一页
42    pub done: bool,
43}
44
45impl<T> Page<T> {
46    /// 恰好一页(拉完就结束)
47    pub fn last(rows: Vec<T>) -> Self {
48        Self {
49            rows,
50            cursor: None,
51            done: true,
52        }
53    }
54
55    /// 还有下一页
56    pub fn more(rows: Vec<T>, cursor: impl Into<String>) -> Self {
57        Self {
58            rows,
59            cursor: Some(cursor.into()),
60            done: false,
61        }
62    }
63}
64
65/// 导出统计
66#[derive(Debug, Clone, Serialize)]
67#[serde(rename_all = "camelCase")]
68pub struct ExportStats {
69    /// 写出行数(不含表头)
70    pub rows: u64,
71    /// 拉取页数
72    pub pages: u64,
73    pub file_size: u64,
74    pub elapsed_ms: u64,
75    pub path: String,
76}
77
78/// 导出执行器
79#[derive(Debug, Clone)]
80pub struct ExportRunner {
81    pub sheet: SheetOptions,
82    /// 解析与写出之间的缓冲页数(越多越吃内存,默认 2)
83    pub queue_pages: usize,
84}
85
86/// 多 sheet 写出线程的消息:行批次 / 本表结束
87enum WriteMsg<T> {
88    Rows(Vec<T>),
89    EndSheet,
90}
91
92impl ExportRunner {
93    pub fn new(sheet: SheetOptions) -> Self {
94        Self {
95            sheet,
96            queue_pages: 2,
97        }
98    }
99
100    pub fn queue_pages(mut self, pages: usize) -> Self {
101        self.queue_pages = pages.max(1);
102        self
103    }
104
105    /// 游标分批拉取 + 流式写文件(推荐:百万行导出走它)
106    pub async fn export_xlsx<T, F, Fut>(
107        &self,
108        path: impl AsRef<Path>,
109        mut fetch: F,
110    ) -> Result<ExportStats>
111    where
112        T: ExcelRow,
113        F: FnMut(Option<String>) -> Fut,
114        Fut: Future<Output = Result<Page<T>>> + Send,
115    {
116        let started = Instant::now();
117        let out_path: PathBuf = path.as_ref().to_path_buf();
118        let (tx, mut rx) = mpsc::channel::<Vec<T>>(self.queue_pages);
119
120        let sheet = self.sheet.clone();
121        let write_path = out_path.clone();
122        let writer_task = tokio::task::spawn_blocking(move || -> Result<u64> {
123            let mut writer = ExcelWriter::new();
124            let mut ws = writer.add_sheet(&sheet)?;
125            ws.write_headers()?;
126            while let Some(rows) = rx.blocking_recv() {
127                for row in &rows {
128                    ws.write_model(row)?;
129                }
130            }
131            ws.finish()?;
132            drop(ws);
133            writer.save(&write_path)
134        });
135
136        let mut cursor: Option<String> = None;
137        let mut pages = 0u64;
138        let mut rows_total = 0u64;
139        let fetch_result: Result<()> = async {
140            loop {
141                let page = fetch(cursor.clone()).await?;
142                pages += 1;
143                let done = page.done;
144                let next = page.cursor.clone();
145                rows_total += page.rows.len() as u64;
146                if !page.rows.is_empty() {
147                    tx.send(page.rows)
148                        .await
149                        .map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
150                }
151                if done {
152                    break;
153                }
154                if next.is_none() {
155                    // 业务没给游标也没标记结束:避免死循环
156                    log::warn!("导出分页既没有 done 也没有 cursor,提前结束");
157                    break;
158                }
159                cursor = next;
160            }
161            Ok(())
162        }
163        .await;
164
165        drop(tx);
166        let file_size = writer_task
167            .await
168            .map_err(|e| ExcelError::Other(format!("导出写出线程异常:{e}")))??;
169        if let Err(e) = fetch_result {
170            // 拉取失败时不要留下半截文件
171            let _ = std::fs::remove_file(&out_path);
172            return Err(e);
173        }
174
175        Ok(ExportStats {
176            rows: rows_total,
177            pages,
178            file_size,
179            elapsed_ms: started.elapsed().as_millis() as u64,
180            path: out_path.to_string_lossy().into_owned(),
181        })
182    }
183
184    /// 多工作表游标导出:每个 sheet 独立分页拉取,顺序写出
185    ///
186    /// 与 [`ExportRunner::export_xlsx`] 的差别只有一个:`fetch` 额外收到
187    /// `sheet_index`,可以给每个 sheet 拉不同筛选条件的数据;
188    /// 写出仍是常量内存,单表百万行不会因为拆表而进内存。
189    pub async fn export_multi_sheet<T, F, Fut>(
190        &self,
191        path: impl AsRef<Path>,
192        sheets: Vec<SheetOptions>,
193        mut fetch: F,
194    ) -> Result<ExportStats>
195    where
196        T: ExcelRow,
197        F: FnMut(usize, Option<String>) -> Fut,
198        Fut: Future<Output = Result<Page<T>>> + Send,
199    {
200        if sheets.is_empty() {
201            return Err(ExcelError::Other("导出至少需要一个工作表".to_string()));
202        }
203        let started = Instant::now();
204        let out_path: PathBuf = path.as_ref().to_path_buf();
205        let sheet_count = sheets.len();
206        let write_path = out_path.clone();
207        let (tx, mut rx) = mpsc::channel::<WriteMsg<T>>(self.queue_pages.max(1));
208
209        // 写线程:每个 sheet 写完才取下一个(只持有一个 SheetWriter 借用)
210        let writer_task = tokio::task::spawn_blocking(move || -> Result<u64> {
211            let mut writer = ExcelWriter::new();
212            for options in &sheets {
213                let mut ws = writer.add_sheet(options)?;
214                ws.write_headers()?;
215                while let Some(WriteMsg::Rows(rows)) = rx.blocking_recv() {
216                    for row in &rows {
217                        ws.write_model(row)?;
218                    }
219                }
220                ws.finish()?;
221            }
222            writer.save(&write_path)
223        });
224
225        let mut pages = 0u64;
226        let mut rows_total = 0u64;
227        let fetch_result: Result<()> = async {
228            for index in 0..sheet_count {
229                let mut cursor: Option<String> = None;
230                loop {
231                    let page = fetch(index, cursor.clone()).await?;
232                    pages += 1;
233                    let done = page.done;
234                    let next = page.cursor.clone();
235                    rows_total += page.rows.len() as u64;
236                    if !page.rows.is_empty() {
237                        tx.send(WriteMsg::Rows(page.rows))
238                            .await
239                            .map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
240                    }
241                    if done {
242                        break;
243                    }
244                    if next.is_none() {
245                        log::warn!("导出分页既没有 done 也没有 cursor,提前结束(sheet {index})");
246                        break;
247                    }
248                    cursor = next;
249                }
250                tx.send(WriteMsg::EndSheet)
251                    .await
252                    .map_err(|_| ExcelError::Other("导出写出任务已结束".to_string()))?;
253            }
254            Ok(())
255        }
256        .await;
257
258        drop(tx);
259        let file_size = writer_task
260            .await
261            .map_err(|e| ExcelError::Other(format!("导出写出线程异常:{e}")))??;
262        if let Err(e) = fetch_result {
263            let _ = std::fs::remove_file(&out_path);
264            return Err(e);
265        }
266
267        Ok(ExportStats {
268            rows: rows_total,
269            pages,
270            file_size,
271            elapsed_ms: started.elapsed().as_millis() as u64,
272            path: out_path.to_string_lossy().into_owned(),
273        })
274    }
275
276    /// 小数据量:一次性写多个工作表(每个 sheet 独立列定义/数据)
277    pub fn export_sheets<T: ExcelRow>(
278        &self,
279        path: impl AsRef<Path>,
280        sheets: Vec<(SheetOptions, Vec<T>)>,
281    ) -> Result<ExportStats> {
282        let started = Instant::now();
283        let out_path = path.as_ref().to_path_buf();
284        let mut rows_total = 0u64;
285        let mut writer = ExcelWriter::new();
286        for (options, rows) in &sheets {
287            let mut ws = writer.add_sheet(options)?;
288            ws.write_headers()?;
289            ws.write_models(rows)?;
290            ws.finish()?;
291            rows_total += rows.len() as u64;
292        }
293        let file_size = writer.save(&out_path)?;
294        Ok(ExportStats {
295            rows: rows_total,
296            pages: sheets.len() as u64,
297            file_size,
298            elapsed_ms: started.elapsed().as_millis() as u64,
299            path: out_path.to_string_lossy().into_owned(),
300        })
301    }
302
303    /// 小数据量:一次性写出文件
304    pub fn export_rows<T: ExcelRow>(&self, path: impl AsRef<Path>, rows: &[T]) -> Result<u64> {
305        let mut writer = ExcelWriter::new();
306        {
307            let mut ws = writer.add_sheet(&self.sheet)?;
308            ws.write_headers()?;
309            ws.write_models(rows)?;
310            ws.finish()?;
311        }
312        writer.save(path)
313    }
314
315    /// 小数据量:写出到内存(HTTP 直接下载)
316    pub fn export_bytes<T: ExcelRow>(&self, rows: &[T]) -> Result<Vec<u8>> {
317        let mut writer = ExcelWriter::new();
318        {
319            let mut ws = writer.add_sheet(&self.sheet)?;
320            ws.write_headers()?;
321            ws.write_models(rows)?;
322            ws.finish()?;
323        }
324        writer.save_to_buffer()
325    }
326}