bulk_append/
bulk_append.rs

1// Copyright 2024 ScopeDB, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod common;
16
17use std::time::Duration;
18
19use scopedb_client::AppendState;
20
21#[tokio::main(flavor = "current_thread")]
22async fn main() -> Result<(), Box<dyn std::error::Error>> {
23    let client = common::client()?;
24    let table = common::write_table(&client)?;
25
26    // The default Stop policy gives strict barriers. Local memory and request
27    // concurrency are bounded independently; concurrent batches have no
28    // defined commit order.
29    let stream = table
30        .append_stream()
31        .target_batch_bytes(32 * 1024)
32        .max_batch_rows(10_000)
33        .flush_interval(Duration::from_secs(1))
34        .max_buffered_bytes(1024 * 1024)
35        .max_concurrent_batches(4)
36        .attempt_timeout(Duration::from_secs(30))
37        .build()?;
38
39    let rows = (1..=10_000).map(|id| {
40        serde_json::json!({
41            "id": id,
42            "occurred_at": jiff::Timestamp::now(),
43            "name": format!("example-{id}"),
44        })
45    });
46
47    let admission_error = match stream.send_all(rows).await {
48        Ok(result) => {
49            println!("accepted locally: {} rows", result.accepted_rows);
50            None
51        }
52        Err(error) => {
53            // There is no stream-wide abort or rollback. Settle the already
54            // accepted prefix even if row production or admission stopped.
55            eprintln!("row admission stopped; settling the accepted prefix: {error}");
56            Some(error)
57        }
58    };
59
60    match stream.shutdown().await {
61        Ok(report) => {
62            println!("committed remotely: {} rows", report.committed_rows);
63            println!("final stream stats: {:?}", stream.stats());
64        }
65        Err(error) => {
66            if matches!(
67                error.append_details().map(|details| details.status),
68                Some(AppendState::Unknown)
69            ) {
70                eprintln!("commit outcome is unknown; reconcile before replaying");
71            } else {
72                eprintln!("bulk append failed: {error}");
73            }
74            eprintln!("final stream stats: {:?}", stream.stats());
75            return Err(error.into());
76        }
77    }
78
79    if let Some(error) = admission_error {
80        return Err(error.into());
81    }
82    Ok(())
83}
bulk_append/bulk_append.rs

bulk_append/
bulk_append.rs
