use crate::Cirrus;
use crate::error::CirrusResult;
use crate::response::{BulkIngestJob, BulkOperation, BulkQueryJob, BulkQueryResults};
use serde::Serialize;
const CSV_CONTENT_TYPE: &str = "text/csv";
const CSV_ACCEPT: &str = "text/csv";
const SFORCE_LOCATOR: &str = "Sforce-Locator";
const SFORCE_NUM_RECORDS: &str = "Sforce-NumberOfRecords";
impl Cirrus {
pub fn bulk(&self) -> BulkHandler<'_> {
BulkHandler { client: self }
}
}
#[derive(Debug)]
pub struct BulkHandler<'a> {
client: &'a Cirrus,
}
impl BulkHandler<'_> {
pub fn ingest(&self) -> BulkIngestHandler<'_> {
BulkIngestHandler {
client: self.client,
}
}
pub fn query(&self) -> BulkQueryHandler<'_> {
BulkQueryHandler {
client: self.client,
}
}
}
#[derive(Debug)]
pub struct BulkIngestHandler<'a> {
client: &'a Cirrus,
}
impl BulkIngestHandler<'_> {
pub async fn create(&self, spec: &BulkIngestSpec) -> CirrusResult<BulkIngestJob> {
self.client.post("jobs/ingest", spec).await
}
pub async fn upload(&self, job_id: &str, csv: bytes::Bytes) -> CirrusResult<()> {
let path = self
.client
.versioned_segments(&["jobs", "ingest", job_id, "batches"])?;
self.client
.send_with_body(reqwest::Method::PUT, &path, csv, CSV_CONTENT_TYPE)
.await
}
pub async fn close(&self, job_id: &str) -> CirrusResult<BulkIngestJob> {
self.patch_state(job_id, "UploadComplete").await
}
pub async fn abort(&self, job_id: &str) -> CirrusResult<BulkIngestJob> {
self.patch_state(job_id, "Aborted").await
}
pub async fn get(&self, job_id: &str) -> CirrusResult<BulkIngestJob> {
let path = self
.client
.versioned_segments(&["jobs", "ingest", job_id])?;
self.client
.send_at::<_, (), ()>(reqwest::Method::GET, &path, None, None)
.await
}
pub async fn delete(&self, job_id: &str) -> CirrusResult<()> {
let path = self
.client
.versioned_segments(&["jobs", "ingest", job_id])?;
self.client
.send_at::<(), (), ()>(reqwest::Method::DELETE, &path, None, None)
.await
}
pub async fn successful_results(&self, job_id: &str) -> CirrusResult<bytes::Bytes> {
self.fetch_csv_results(job_id, "successfulResults").await
}
pub async fn failed_results(&self, job_id: &str) -> CirrusResult<bytes::Bytes> {
self.fetch_csv_results(job_id, "failedResults").await
}
pub async fn unprocessed_records(&self, job_id: &str) -> CirrusResult<bytes::Bytes> {
self.fetch_csv_results(job_id, "unprocessedrecords").await
}
async fn patch_state(&self, job_id: &str, new_state: &str) -> CirrusResult<BulkIngestJob> {
let path = self
.client
.versioned_segments(&["jobs", "ingest", job_id])?;
let body = StatePatch { state: new_state };
self.client
.send_at::<_, (), _>(reqwest::Method::PATCH, &path, None, Some(&body))
.await
}
async fn fetch_csv_results(&self, job_id: &str, kind: &str) -> CirrusResult<bytes::Bytes> {
let path = self
.client
.versioned_segments(&["jobs", "ingest", job_id, kind])?;
let (_, bytes) = self
.client
.fetch_raw(reqwest::Method::GET, &path, CSV_ACCEPT, None)
.await?;
Ok(bytes)
}
}
#[derive(Debug)]
pub struct BulkQueryHandler<'a> {
client: &'a Cirrus,
}
impl BulkQueryHandler<'_> {
pub async fn create(&self, spec: &BulkQuerySpec) -> CirrusResult<BulkQueryJob> {
self.client.post("jobs/query", spec).await
}
pub async fn get(&self, job_id: &str) -> CirrusResult<BulkQueryJob> {
let path = self.client.versioned_segments(&["jobs", "query", job_id])?;
self.client
.send_at::<_, (), ()>(reqwest::Method::GET, &path, None, None)
.await
}
pub async fn abort(&self, job_id: &str) -> CirrusResult<BulkQueryJob> {
let path = self.client.versioned_segments(&["jobs", "query", job_id])?;
let body = StatePatch { state: "Aborted" };
self.client
.send_at::<_, (), _>(reqwest::Method::PATCH, &path, None, Some(&body))
.await
}
pub async fn results(
&self,
job_id: &str,
locator: Option<&str>,
max_records: Option<u32>,
) -> CirrusResult<BulkQueryResults> {
let path = self
.client
.versioned_segments(&["jobs", "query", job_id, "results"])?;
let max_records_str = max_records.map(|n| n.to_string());
let mut query: Vec<(&str, &str)> = Vec::with_capacity(2);
if let Some(loc) = locator {
query.push(("locator", loc));
}
if let Some(ref n) = max_records_str {
query.push(("maxRecords", n.as_str()));
}
let query_slice = if query.is_empty() {
None
} else {
Some(query.as_slice())
};
let (headers, csv) = self
.client
.fetch_raw(reqwest::Method::GET, &path, CSV_ACCEPT, query_slice)
.await?;
Ok(BulkQueryResults {
csv,
locator: header_string(&headers, SFORCE_LOCATOR).filter(|s| s != "null"),
number_of_records: header_string(&headers, SFORCE_NUM_RECORDS)
.and_then(|s| s.parse().ok()),
})
}
pub async fn delete(&self, job_id: &str) -> CirrusResult<()> {
let path = self.client.versioned_segments(&["jobs", "query", job_id])?;
self.client
.send_at::<(), (), ()>(reqwest::Method::DELETE, &path, None, None)
.await
}
}
#[derive(Debug, Clone, Serialize)]
pub struct BulkIngestSpec {
pub object: String,
pub operation: BulkOperation,
#[serde(
rename = "externalIdFieldName",
skip_serializing_if = "Option::is_none"
)]
pub external_id_field_name: Option<String>,
#[serde(rename = "lineEnding", skip_serializing_if = "Option::is_none")]
pub line_ending: Option<crate::response::BulkLineEnding>,
#[serde(rename = "columnDelimiter", skip_serializing_if = "Option::is_none")]
pub column_delimiter: Option<crate::response::BulkColumnDelimiter>,
#[serde(rename = "assignmentRuleId", skip_serializing_if = "Option::is_none")]
pub assignment_rule_id: Option<String>,
}
#[derive(Debug, Clone, Serialize)]
pub struct BulkQuerySpec {
pub query: String,
pub operation: BulkOperation,
#[serde(rename = "lineEnding", skip_serializing_if = "Option::is_none")]
pub line_ending: Option<crate::response::BulkLineEnding>,
#[serde(rename = "columnDelimiter", skip_serializing_if = "Option::is_none")]
pub column_delimiter: Option<crate::response::BulkColumnDelimiter>,
}
#[derive(Serialize)]
struct StatePatch<'a> {
state: &'a str,
}
fn header_string(headers: &reqwest::header::HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|v| v.to_str().ok())
.map(str::to_owned)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
use crate::auth::StaticTokenAuth;
use crate::response::{BulkColumnDelimiter, BulkJobState, BulkLineEnding};
use serde_json::json;
use std::sync::Arc;
use wiremock::matchers::{body_json, header, method, path, query_param};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn fixture(uri: String) -> Cirrus {
let auth = Arc::new(StaticTokenAuth::new("tok", uri));
Cirrus::builder().auth(auth).build().unwrap()
}
fn ingest_job_response(id: &str, state: &str) -> serde_json::Value {
json!({
"id": id,
"operation": "insert",
"object": "Account",
"createdById": "005xx",
"createdDate": "2024-01-01T00:00:00.000+0000",
"systemModstamp": "2024-01-01T00:00:00.000+0000",
"state": state,
"concurrencyMode": "Parallel",
"contentType": "CSV",
"apiVersion": 60.0,
"lineEnding": "LF",
"columnDelimiter": "COMMA",
"jobType": "V2Ingest"
})
}
#[tokio::test]
async fn ingest_create_posts_spec_and_parses_open_job() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/services/data/v66.0/jobs/ingest"))
.and(header("authorization", "Bearer tok"))
.and(body_json(json!({
"object": "Account",
"operation": "insert"
})))
.respond_with(
ResponseTemplate::new(200).set_body_json(ingest_job_response("750xx", "Open")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf
.bulk()
.ingest()
.create(&BulkIngestSpec {
object: "Account".into(),
operation: BulkOperation::Insert,
external_id_field_name: None,
line_ending: None,
column_delimiter: None,
assignment_rule_id: None,
})
.await
.unwrap();
assert_eq!(job.id, "750xx");
assert_eq!(job.state, BulkJobState::Open);
}
#[tokio::test]
async fn ingest_create_serializes_upsert_with_external_id() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/services/data/v66.0/jobs/ingest"))
.and(body_json(json!({
"object": "Account",
"operation": "upsert",
"externalIdFieldName": "External_Id__c",
"lineEnding": "CRLF",
"columnDelimiter": "TAB"
})))
.respond_with(
ResponseTemplate::new(200).set_body_json(ingest_job_response("750xx", "Open")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
sf.bulk()
.ingest()
.create(&BulkIngestSpec {
object: "Account".into(),
operation: BulkOperation::Upsert,
external_id_field_name: Some("External_Id__c".into()),
line_ending: Some(BulkLineEnding::CRLF),
column_delimiter: Some(BulkColumnDelimiter::Tab),
assignment_rule_id: None,
})
.await
.unwrap();
}
#[tokio::test]
async fn ingest_upload_sends_csv_body_with_text_csv_content_type() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.and(path("/services/data/v66.0/jobs/ingest/750xx/batches"))
.and(header("content-type", "text/csv"))
.and(header("authorization", "Bearer tok"))
.respond_with(ResponseTemplate::new(201))
.mount(&server)
.await;
let sf = fixture(server.uri());
let csv = bytes::Bytes::from_static(b"Name\nAcme\nGlobex\n");
sf.bulk().ingest().upload("750xx", csv).await.unwrap();
}
#[tokio::test]
async fn ingest_close_patches_state_to_upload_complete() {
let server = MockServer::start().await;
Mock::given(method("PATCH"))
.and(path("/services/data/v66.0/jobs/ingest/750xx"))
.and(body_json(json!({"state": "UploadComplete"})))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(ingest_job_response("750xx", "UploadComplete")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf.bulk().ingest().close("750xx").await.unwrap();
assert_eq!(job.state, BulkJobState::UploadComplete);
}
#[tokio::test]
async fn ingest_abort_patches_state_to_aborted() {
let server = MockServer::start().await;
Mock::given(method("PATCH"))
.and(path("/services/data/v66.0/jobs/ingest/750xx"))
.and(body_json(json!({"state": "Aborted"})))
.respond_with(
ResponseTemplate::new(200).set_body_json(ingest_job_response("750xx", "Aborted")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf.bulk().ingest().abort("750xx").await.unwrap();
assert_eq!(job.state, BulkJobState::Aborted);
}
#[tokio::test]
async fn ingest_get_returns_completed_job_with_metrics() {
let server = MockServer::start().await;
let mut completed = ingest_job_response("750xx", "JobComplete");
completed["numberRecordsProcessed"] = json!(100);
completed["numberRecordsFailed"] = json!(2);
completed["totalProcessingTime"] = json!(2349);
Mock::given(method("GET"))
.and(path("/services/data/v66.0/jobs/ingest/750xx"))
.respond_with(ResponseTemplate::new(200).set_body_json(completed))
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf.bulk().ingest().get("750xx").await.unwrap();
assert_eq!(job.state, BulkJobState::JobComplete);
assert_eq!(job.number_records_processed, Some(100));
assert_eq!(job.number_records_failed, Some(2));
}
#[tokio::test]
async fn ingest_delete_returns_204() {
let server = MockServer::start().await;
Mock::given(method("DELETE"))
.and(path("/services/data/v66.0/jobs/ingest/750xx"))
.respond_with(ResponseTemplate::new(204))
.mount(&server)
.await;
let sf = fixture(server.uri());
sf.bulk().ingest().delete("750xx").await.unwrap();
}
#[tokio::test]
async fn ingest_successful_results_returns_csv_bytes() {
let server = MockServer::start().await;
let csv = "sf__Id,sf__Created,Name\n001xx,true,Acme\n";
Mock::given(method("GET"))
.and(path(
"/services/data/v66.0/jobs/ingest/750xx/successfulResults",
))
.and(header("accept", "text/csv"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(csv)
.insert_header("content-type", "text/csv"),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let bytes = sf
.bulk()
.ingest()
.successful_results("750xx")
.await
.unwrap();
assert_eq!(&bytes[..], csv.as_bytes());
}
#[tokio::test]
async fn ingest_failed_results_surfaces_csv_error_rows() {
let server = MockServer::start().await;
let csv = "sf__Id,sf__Error,Name\n,REQUIRED_FIELD_MISSING:Name,\n";
Mock::given(method("GET"))
.and(path("/services/data/v66.0/jobs/ingest/750xx/failedResults"))
.respond_with(ResponseTemplate::new(200).set_body_string(csv))
.mount(&server)
.await;
let sf = fixture(server.uri());
let bytes = sf.bulk().ingest().failed_results("750xx").await.unwrap();
assert!(bytes.starts_with(b"sf__Id,sf__Error"));
}
#[tokio::test]
async fn ingest_results_path_404_surfaces_as_api_error() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path(
"/services/data/v66.0/jobs/ingest/missing/successfulResults",
))
.respond_with(ResponseTemplate::new(400).set_body_json(json!([{
"message": "InvalidJob: Bulk API job missing not found",
"errorCode": "INVALIDJOBSTATE"
}])))
.mount(&server)
.await;
let sf = fixture(server.uri());
let err = sf
.bulk()
.ingest()
.successful_results("missing")
.await
.unwrap_err();
match err {
crate::CirrusError::Api { status, errors, .. } => {
assert_eq!(status, 400);
assert_eq!(errors[0].error_code, "INVALIDJOBSTATE");
}
other => panic!("expected Api error, got {other:?}"),
}
}
fn query_job_response(id: &str, state: &str) -> serde_json::Value {
json!({
"id": id,
"operation": "query",
"state": state,
"object": "Account",
"createdById": "005xx",
"createdDate": "2024-01-01T00:00:00.000+0000",
"systemModstamp": "2024-01-01T00:00:00.000+0000",
"concurrencyMode": "Parallel",
"contentType": "CSV",
"apiVersion": 60.0,
"lineEnding": "LF",
"columnDelimiter": "COMMA"
})
}
#[tokio::test]
async fn query_create_posts_soql() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/services/data/v66.0/jobs/query"))
.and(body_json(json!({
"query": "SELECT Id, Name FROM Account",
"operation": "query"
})))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(query_job_response("750xx", "UploadComplete")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf
.bulk()
.query()
.create(&BulkQuerySpec {
query: "SELECT Id, Name FROM Account".into(),
operation: BulkOperation::Query,
line_ending: None,
column_delimiter: None,
})
.await
.unwrap();
assert_eq!(job.state, BulkJobState::UploadComplete);
}
#[tokio::test]
async fn query_results_returns_csv_with_locator_header() {
let server = MockServer::start().await;
let csv = "Id,Name\n001xx,Acme\n";
Mock::given(method("GET"))
.and(path("/services/data/v66.0/jobs/query/750xx/results"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(csv)
.insert_header("Sforce-Locator", "MTAwMA")
.insert_header("Sforce-NumberOfRecords", "1"),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let result = sf
.bulk()
.query()
.results("750xx", None, None)
.await
.unwrap();
assert_eq!(&result.csv[..], csv.as_bytes());
assert_eq!(result.locator.as_deref(), Some("MTAwMA"));
assert_eq!(result.number_of_records, Some(1));
}
#[tokio::test]
async fn query_results_treats_null_locator_as_done() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/services/data/v66.0/jobs/query/750xx/results"))
.and(query_param("locator", "MTAwMA"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("")
.insert_header("Sforce-Locator", "null")
.insert_header("Sforce-NumberOfRecords", "0"),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let result = sf
.bulk()
.query()
.results("750xx", Some("MTAwMA"), None)
.await
.unwrap();
assert!(result.locator.is_none());
assert_eq!(result.number_of_records, Some(0));
}
#[tokio::test]
async fn query_results_serializes_max_records() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/services/data/v66.0/jobs/query/750xx/results"))
.and(query_param("maxRecords", "10000"))
.respond_with(ResponseTemplate::new(200).set_body_string("Id\n"))
.mount(&server)
.await;
let sf = fixture(server.uri());
sf.bulk()
.query()
.results("750xx", None, Some(10000))
.await
.unwrap();
}
#[tokio::test]
async fn query_abort_patches_state() {
let server = MockServer::start().await;
Mock::given(method("PATCH"))
.and(path("/services/data/v66.0/jobs/query/750xx"))
.and(body_json(json!({"state": "Aborted"})))
.respond_with(
ResponseTemplate::new(200).set_body_json(query_job_response("750xx", "Aborted")),
)
.mount(&server)
.await;
let sf = fixture(server.uri());
let job = sf.bulk().query().abort("750xx").await.unwrap();
assert_eq!(job.state, BulkJobState::Aborted);
}
}