1use crate::error::{map_sqlx_error, AwaError};
2use crate::job::{InsertOpts, InsertParams, JobRow, JobState};
3use crate::unique::compute_unique_key;
4use crate::JobArgs;
5use sqlx::postgres::PgConnection;
6use sqlx::{PgExecutor, PgPool};
7
8const COPY_NULL_SENTINEL: &str = "__AWA_NULL__";
9const INSERT_COMPAT_SQL: &str = r#"
10 SELECT *
11 FROM awa.insert_job_compat(
12 $1,
13 $2,
14 $3,
15 $4::awa.job_state,
16 $5,
17 $6,
18 $7,
19 $8,
20 $9,
21 $10,
22 $11::text::bit(8),
23 $12
24 )
25"#;
26
27pub(crate) const POSTGRES_INSERT_JOB_SQL: &str = r#"
38 SELECT *, state::text AS state_str
39 FROM awa.insert_job_compat(
40 $1,
41 $2,
42 $3,
43 $4::text::awa.job_state,
44 $5::smallint,
45 $6::smallint,
46 $7,
47 $8,
48 $9::text[],
49 $10,
50 $11::text::bit(8),
51 $12
52 )
53"#;
54
55pub(crate) fn reject_null_bytes(value: &serde_json::Value) -> Result<(), AwaError> {
68 match value {
69 serde_json::Value::String(s) if s.contains('\0') => Err(AwaError::Validation(
70 "job args/metadata must not contain null bytes (\\u0000): Postgres JSONB does not support them".into(),
71 )),
72 serde_json::Value::Array(arr) => {
73 for v in arr {
74 reject_null_bytes(v)?;
75 }
76 Ok(())
77 }
78 serde_json::Value::Object(map) => {
79 for (k, v) in map {
80 if k.contains('\0') {
81 return Err(AwaError::Validation(
82 "job args/metadata keys must not contain null bytes (\\u0000)".into(),
83 ));
84 }
85 reject_null_bytes(v)?;
86 }
87 Ok(())
88 }
89 _ => Ok(()),
90 }
91}
92
93#[derive(Debug, Clone)]
99pub struct PreparedJobInsert {
100 pub(crate) kind: String,
101 pub(crate) queue: String,
102 pub(crate) args: serde_json::Value,
103 pub(crate) state: JobState,
104 pub(crate) priority: i16,
105 pub(crate) max_attempts: i16,
106 pub(crate) run_at: Option<chrono::DateTime<chrono::Utc>>,
107 pub(crate) metadata: serde_json::Value,
108 pub(crate) tags: Vec<String>,
109 pub(crate) unique_key: Option<Vec<u8>>,
110 pub(crate) unique_states: Option<String>,
111 pub(crate) ordering_key: Option<Vec<u8>>,
112}
113
114pub(crate) type PreparedRow = PreparedJobInsert;
115
116impl PreparedJobInsert {
117 pub fn kind(&self) -> &str {
119 &self.kind
120 }
121
122 pub fn queue(&self) -> &str {
124 &self.queue
125 }
126
127 pub fn args(&self) -> &serde_json::Value {
129 &self.args
130 }
131
132 pub fn state(&self) -> JobState {
134 self.state
135 }
136
137 pub fn state_db_str(&self) -> &'static str {
139 self.state.as_str()
140 }
141
142 pub fn priority(&self) -> i16 {
144 self.priority
145 }
146
147 pub fn max_attempts(&self) -> i16 {
149 self.max_attempts
150 }
151
152 pub fn run_at(&self) -> Option<chrono::DateTime<chrono::Utc>> {
154 self.run_at
155 }
156
157 pub fn metadata(&self) -> &serde_json::Value {
159 &self.metadata
160 }
161
162 pub fn tags(&self) -> &[String] {
164 &self.tags
165 }
166
167 pub fn unique_key(&self) -> Option<&[u8]> {
169 self.unique_key.as_deref()
170 }
171
172 pub fn unique_states_bit_string(&self) -> Option<&str> {
174 self.unique_states.as_deref()
175 }
176
177 pub fn ordering_key(&self) -> Option<&[u8]> {
179 self.ordering_key.as_deref()
180 }
181}
182
183fn build_multi_insert_query(count: usize) -> String {
184 let mut query = String::from(
185 "WITH input (ord, kind, queue, args, state, priority, max_attempts, run_at, metadata, tags, unique_key, unique_states, ordering_key) AS (VALUES ",
186 );
187
188 let params_per_row = 12u32;
189 let mut param_index = 1u32;
190 for i in 0..count {
191 if i > 0 {
192 query.push_str(", ");
193 }
194 query.push_str(&format!(
195 "({}, ${}, ${}::text, ${}::jsonb, ${}::awa.job_state, ${}::smallint, ${}::smallint, ${}::timestamptz, ${}::jsonb, ${}::text[], ${}, ${}::bit(8), ${})",
196 i,
197 param_index,
198 param_index + 1,
199 param_index + 2,
200 param_index + 3,
201 param_index + 4,
202 param_index + 5,
203 param_index + 6,
204 param_index + 7,
205 param_index + 8,
206 param_index + 9,
207 param_index + 10,
208 param_index + 11,
209 ));
210 param_index += params_per_row;
211 }
212 query.push_str(
213 ") \
214 SELECT inserted.* \
215 FROM input \
216 CROSS JOIN LATERAL awa.insert_job_compat(\
217 input.kind, \
218 input.queue, \
219 input.args, \
220 input.state, \
221 input.priority, \
222 input.max_attempts, \
223 input.run_at, \
224 input.metadata, \
225 input.tags, \
226 input.unique_key, \
227 input.unique_states, \
228 input.ordering_key\
229 ) AS inserted \
230 ORDER BY input.ord",
231 );
232 query
233}
234
235fn compute_unique_fields(
237 kind: &str,
238 args: &serde_json::Value,
239 opts: &InsertOpts,
240) -> (Option<Vec<u8>>, Option<String>) {
241 let unique_key = opts.unique.as_ref().map(|u| {
242 compute_unique_key(
243 kind,
244 if u.by_queue { Some(&opts.queue) } else { None },
245 if u.by_args { Some(args) } else { None },
246 u.by_period,
247 )
248 });
249
250 let unique_states = opts.unique.as_ref().map(|u| {
251 let mut bit_string = String::with_capacity(8);
255 for bit_position in 0..8 {
256 if u.states & (1 << bit_position) != 0 {
257 bit_string.push('1');
258 } else {
259 bit_string.push('0');
260 }
261 }
262 bit_string
263 });
264
265 (unique_key, unique_states)
266}
267
268pub fn prepare_job_insert(
274 args: &impl JobArgs,
275 opts: InsertOpts,
276) -> Result<PreparedJobInsert, AwaError> {
277 let kind = args.kind_str().to_string();
278 let args_value = args.to_args()?;
279 prepare_raw_job_insert(kind, args_value, opts)
280}
281
282pub fn prepare_raw_job_insert(
287 kind: impl Into<String>,
288 args: impl Into<serde_json::Value>,
289 opts: InsertOpts,
290) -> Result<PreparedJobInsert, AwaError> {
291 let kind = kind.into();
292 let args = args.into();
293
294 reject_null_bytes(&args)?;
295 reject_null_bytes(&opts.metadata)?;
296
297 let state = if opts.run_at.is_some() {
298 JobState::Scheduled
299 } else {
300 JobState::Available
301 };
302
303 let (unique_key, unique_states) = compute_unique_fields(&kind, &args, &opts);
304
305 Ok(PreparedJobInsert {
306 kind,
307 queue: opts.queue,
308 args,
309 state,
310 priority: opts.priority,
311 max_attempts: opts.max_attempts,
312 run_at: opts.run_at,
313 metadata: opts.metadata,
314 tags: opts.tags,
315 unique_key,
316 unique_states,
317 ordering_key: opts.ordering_key,
318 })
319}
320
321pub(crate) fn prepare_row(args: &impl JobArgs, opts: InsertOpts) -> Result<PreparedRow, AwaError> {
323 prepare_job_insert(args, opts)
324}
325
326pub(crate) fn prepare_row_raw(
328 kind: String,
329 args: serde_json::Value,
330 opts: InsertOpts,
331) -> Result<PreparedRow, AwaError> {
332 prepare_raw_job_insert(kind, args, opts)
333}
334
335pub async fn insert<'e, E>(executor: E, args: &impl JobArgs) -> Result<JobRow, AwaError>
339where
340 E: PgExecutor<'e>,
341{
342 insert_with(executor, args, InsertOpts::default()).await
343}
344
345#[tracing::instrument(skip(executor, args), fields(job.kind = args.kind_str(), job.queue = %opts.queue))]
347pub async fn insert_with<'e, E>(
348 executor: E,
349 args: &impl JobArgs,
350 opts: InsertOpts,
351) -> Result<JobRow, AwaError>
352where
353 E: PgExecutor<'e>,
354{
355 let row = prepare_row(args, opts)?;
356 sqlx::query_as::<_, JobRow>(INSERT_COMPAT_SQL)
357 .bind(&row.kind)
358 .bind(&row.queue)
359 .bind(&row.args)
360 .bind(row.state)
361 .bind(row.priority)
362 .bind(row.max_attempts)
363 .bind(row.run_at)
364 .bind(&row.metadata)
365 .bind(&row.tags)
366 .bind(&row.unique_key)
367 .bind(&row.unique_states)
368 .bind(&row.ordering_key)
369 .fetch_one(executor)
370 .await
371 .map_err(map_sqlx_error)
372}
373
374fn precompute_rows(jobs: &[InsertParams]) -> Result<Vec<PreparedRow>, AwaError> {
376 jobs.iter()
377 .map(|job| prepare_row_raw(job.kind.clone(), job.args.clone(), job.opts.clone()))
378 .collect()
379}
380
381#[tracing::instrument(skip(executor, jobs), fields(job.count = jobs.len()))]
386pub async fn insert_many<'e, E>(executor: E, jobs: &[InsertParams]) -> Result<Vec<JobRow>, AwaError>
387where
388 E: PgExecutor<'e>,
389{
390 if jobs.is_empty() {
391 return Ok(Vec::new());
392 }
393
394 let rows = precompute_rows(jobs)?;
395 let query = build_multi_insert_query(rows.len());
396
397 let mut sql_query = sqlx::query_as::<_, JobRow>(&query);
398
399 for row in &rows {
400 sql_query = sql_query
401 .bind(&row.kind)
402 .bind(&row.queue)
403 .bind(&row.args)
404 .bind(row.state)
405 .bind(row.priority)
406 .bind(row.max_attempts)
407 .bind(row.run_at)
408 .bind(&row.metadata)
409 .bind(&row.tags)
410 .bind(&row.unique_key)
411 .bind(&row.unique_states)
412 .bind(&row.ordering_key);
413 }
414
415 let results = sql_query.fetch_all(executor).await?;
416
417 Ok(results)
418}
419
420#[tracing::instrument(skip(conn, jobs), fields(job.count = jobs.len()))]
429pub async fn insert_many_copy(
430 conn: &mut PgConnection,
431 jobs: &[InsertParams],
432) -> Result<Vec<JobRow>, AwaError> {
433 if jobs.is_empty() {
434 return Ok(Vec::new());
435 }
436
437 let rows = precompute_rows(jobs)?;
438
439 sqlx::query(
445 r#"
446 CREATE TEMP TABLE IF NOT EXISTS pg_temp.awa_copy_staging (
447 kind TEXT NOT NULL,
448 queue TEXT NOT NULL,
449 args JSONB NOT NULL,
450 state awa.job_state NOT NULL,
451 priority SMALLINT NOT NULL,
452 max_attempts SMALLINT NOT NULL,
453 run_at TIMESTAMPTZ,
454 metadata JSONB NOT NULL,
455 tags TEXT[] NOT NULL,
456 unique_key BYTEA,
457 unique_states BIT(8),
458 ordering_key BYTEA
459 ) ON COMMIT DELETE ROWS
460 "#,
461 )
462 .execute(&mut *conn)
463 .await?;
464
465 let mut csv_buf = Vec::with_capacity(rows.len() * 256);
467 for row in &rows {
468 write_csv_row(&mut csv_buf, row);
469 }
470
471 let mut copy_in = conn
472 .copy_in_raw(
473 "COPY pg_temp.awa_copy_staging (kind, queue, args, state, priority, max_attempts, run_at, metadata, tags, unique_key, unique_states, ordering_key) FROM STDIN WITH (FORMAT csv, NULL '__AWA_NULL__')",
474 )
475 .await?;
476 copy_in.send(csv_buf).await?;
477 copy_in.finish().await?;
478
479 let has_unique = rows.iter().any(|r| r.unique_key.is_some());
481
482 let results = if has_unique {
483 let staged_rows = sqlx::query_as::<
488 _,
489 (
490 String,
491 String,
492 serde_json::Value,
493 String,
494 i16,
495 i16,
496 Option<chrono::DateTime<chrono::Utc>>,
497 serde_json::Value,
498 Vec<String>,
499 Option<Vec<u8>>,
500 Option<String>,
501 Option<Vec<u8>>,
502 ),
503 >(
504 r#"
505 SELECT
506 kind,
507 queue,
508 args,
509 state::text,
510 priority,
511 max_attempts,
512 run_at,
513 metadata,
514 tags,
515 unique_key,
516 unique_states::text,
517 ordering_key
518 FROM pg_temp.awa_copy_staging
519 "#,
520 )
521 .fetch_all(&mut *conn)
522 .await?;
523
524 let mut inserted = Vec::with_capacity(staged_rows.len());
525 for (
526 kind,
527 queue,
528 args,
529 state,
530 priority,
531 max_attempts,
532 run_at,
533 metadata,
534 tags,
535 unique_key,
536 unique_states,
537 ordering_key,
538 ) in staged_rows
539 {
540 sqlx::query("SAVEPOINT awa_copy_unique_row")
541 .execute(&mut *conn)
542 .await?;
543
544 let result = sqlx::query_as::<_, JobRow>(POSTGRES_INSERT_JOB_SQL)
545 .bind(&kind)
546 .bind(&queue)
547 .bind(&args)
548 .bind(&state)
549 .bind(priority)
550 .bind(max_attempts)
551 .bind(run_at)
552 .bind(&metadata)
553 .bind(&tags)
554 .bind(&unique_key)
555 .bind(&unique_states)
556 .bind(&ordering_key)
557 .fetch_one(&mut *conn)
558 .await;
559
560 match result {
561 Ok(row) => {
562 inserted.push(row);
563 sqlx::query("RELEASE SAVEPOINT awa_copy_unique_row")
564 .execute(&mut *conn)
565 .await?;
566 }
567 Err(sqlx::Error::Database(db_err)) if db_err.code().as_deref() == Some("23505") => {
568 sqlx::query("ROLLBACK TO SAVEPOINT awa_copy_unique_row")
569 .execute(&mut *conn)
570 .await?;
571 sqlx::query("RELEASE SAVEPOINT awa_copy_unique_row")
572 .execute(&mut *conn)
573 .await?;
574 continue;
575 }
576 Err(err) => {
577 sqlx::query("ROLLBACK TO SAVEPOINT awa_copy_unique_row")
578 .execute(&mut *conn)
579 .await?;
580 sqlx::query("RELEASE SAVEPOINT awa_copy_unique_row")
581 .execute(&mut *conn)
582 .await?;
583 return Err(AwaError::Database(err));
584 }
585 }
586 }
587
588 inserted
589 } else {
590 let insert_sql = r#"
591 WITH staged AS (
592 SELECT
593 *,
594 row_number() OVER () AS ord
595 FROM pg_temp.awa_copy_staging
596 )
597 SELECT inserted.*
598 FROM staged
599 CROSS JOIN LATERAL awa.insert_job_compat(
600 staged.kind,
601 staged.queue,
602 staged.args,
603 staged.state,
604 staged.priority,
605 staged.max_attempts,
606 staged.run_at,
607 staged.metadata,
608 staged.tags,
609 staged.unique_key,
610 staged.unique_states,
611 staged.ordering_key
612 ) AS inserted
613 ORDER BY staged.ord
614 "#;
615
616 sqlx::query_as::<_, JobRow>(insert_sql)
617 .fetch_all(&mut *conn)
618 .await?
619 };
620
621 sqlx::query("DELETE FROM pg_temp.awa_copy_staging")
624 .execute(&mut *conn)
625 .await?;
626
627 Ok(results)
628}
629
630#[tracing::instrument(skip(pool, jobs), fields(job.count = jobs.len()))]
635pub async fn insert_many_copy_from_pool(
636 pool: &PgPool,
637 jobs: &[InsertParams],
638) -> Result<Vec<JobRow>, AwaError> {
639 if jobs.is_empty() {
640 return Ok(Vec::new());
641 }
642
643 let mut tx = pool.begin().await?;
644 let results = insert_many_copy(&mut tx, jobs).await?;
645 tx.commit().await?;
646
647 Ok(results)
648}
649
650fn write_csv_row(buf: &mut Vec<u8>, row: &PreparedRow) {
654 write_csv_field(buf, &row.kind);
656 buf.push(b',');
657 write_csv_field(buf, &row.queue);
659 buf.push(b',');
660 let args_str = serde_json::to_string(&row.args).expect("JSON serialization should not fail");
662 write_csv_field(buf, &args_str);
663 buf.push(b',');
664 write_csv_field(buf, &row.state.to_string());
666 buf.push(b',');
667 buf.extend_from_slice(row.priority.to_string().as_bytes());
669 buf.push(b',');
670 buf.extend_from_slice(row.max_attempts.to_string().as_bytes());
672 buf.push(b',');
673 match &row.run_at {
675 Some(dt) => write_csv_field(buf, &dt.to_rfc3339()),
676 None => buf.extend_from_slice(COPY_NULL_SENTINEL.as_bytes()),
677 }
678 buf.push(b',');
679 let metadata_str =
681 serde_json::to_string(&row.metadata).expect("JSON serialization should not fail");
682 write_csv_field(buf, &metadata_str);
683 buf.push(b',');
684 write_pg_text_array(buf, &row.tags);
686 buf.push(b',');
687 match &row.unique_key {
689 Some(key) => {
690 let bytea_hex = format!("\\x{}", hex::encode(key));
691 write_csv_field(buf, &bytea_hex);
692 }
693 None => buf.extend_from_slice(COPY_NULL_SENTINEL.as_bytes()),
694 }
695 buf.push(b',');
696 match &row.unique_states {
698 Some(bits) => write_csv_field(buf, bits),
699 None => buf.extend_from_slice(COPY_NULL_SENTINEL.as_bytes()),
700 }
701 buf.push(b',');
702 match &row.ordering_key {
704 Some(key) => {
705 let bytea_hex = format!("\\x{}", hex::encode(key));
706 write_csv_field(buf, &bytea_hex);
707 }
708 None => buf.extend_from_slice(COPY_NULL_SENTINEL.as_bytes()),
709 }
710 buf.push(b'\n');
711}
712
713fn write_csv_field(buf: &mut Vec<u8>, value: &str) {
715 if value.contains(',')
716 || value.contains('"')
717 || value.contains('\n')
718 || value.contains('\r')
719 || value.contains('\\')
720 || value == COPY_NULL_SENTINEL
721 {
722 buf.push(b'"');
723 for byte in value.bytes() {
724 if byte == b'"' {
725 buf.push(b'"');
726 }
727 buf.push(byte);
728 }
729 buf.push(b'"');
730 } else {
731 buf.extend_from_slice(value.as_bytes());
732 }
733}
734
735fn write_pg_text_array(buf: &mut Vec<u8>, values: &[String]) {
738 buf.push(b'"');
739 buf.push(b'{');
740 for (i, val) in values.iter().enumerate() {
741 if i > 0 {
742 buf.push(b',');
743 }
744 if val.is_empty()
745 || val.contains(',')
746 || val.contains('"')
747 || val.contains('\\')
748 || val.contains('{')
749 || val.contains('}')
750 || val.contains(' ')
751 || val.eq_ignore_ascii_case("NULL")
752 {
753 buf.push(b'"');
754 buf.push(b'"');
755 for ch in val.chars() {
756 match ch {
757 '"' => buf.extend_from_slice(b"\\\"\""),
758 '\\' => buf.extend_from_slice(b"\\\\"),
759 _ => {
760 let mut utf8_buf = [0u8; 4];
761 buf.extend_from_slice(ch.encode_utf8(&mut utf8_buf).as_bytes());
762 }
763 }
764 }
765 buf.push(b'"');
766 buf.push(b'"');
767 } else {
768 buf.extend_from_slice(val.as_bytes());
769 }
770 }
771 buf.push(b'}');
772 buf.push(b'"');
773}
774
775pub fn params(args: &impl JobArgs) -> Result<InsertParams, AwaError> {
777 params_with(args, InsertOpts::default())
778}
779
780pub fn params_with(args: &impl JobArgs, opts: InsertOpts) -> Result<InsertParams, AwaError> {
782 Ok(InsertParams {
783 kind: args.kind_str().to_string(),
784 args: args.to_args()?,
785 opts,
786 })
787}
788
789#[cfg(test)]
790mod tests {
791 use super::*;
792 use crate::job::UniqueOpts;
793 use chrono::TimeZone;
794
795 #[derive(serde::Serialize)]
796 struct SendEmail {
797 to: String,
798 subject: String,
799 }
800
801 impl JobArgs for SendEmail {
802 fn kind() -> &'static str {
803 "send_email"
804 }
805 }
806
807 #[test]
808 fn prepare_job_insert_exposes_canonical_adapter_values() {
809 let run_at = chrono::Utc.with_ymd_and_hms(2026, 5, 15, 12, 0, 0).unwrap();
810 let prepared = prepare_job_insert(
811 &SendEmail {
812 to: "ada@example.com".to_string(),
813 subject: "hello".to_string(),
814 },
815 InsertOpts {
816 queue: "email".to_string(),
817 priority: 1,
818 max_attempts: 3,
819 run_at: Some(run_at),
820 metadata: serde_json::json!({"source": "test"}),
821 tags: vec!["welcome".to_string(), "external".to_string()],
822 unique: Some(UniqueOpts {
823 by_queue: true,
824 by_args: true,
825 by_period: Some(60),
826 states: 0b1000_0001,
827 }),
828 ordering_key: Some(vec![1, 2, 3]),
829 ..Default::default()
830 },
831 )
832 .unwrap();
833
834 assert_eq!(prepared.kind(), "send_email");
835 assert_eq!(prepared.queue(), "email");
836 assert_eq!(prepared.args()["to"], "ada@example.com");
837 assert_eq!(prepared.state(), JobState::Scheduled);
838 assert_eq!(prepared.state_db_str(), "scheduled");
839 assert_eq!(prepared.priority(), 1);
840 assert_eq!(prepared.max_attempts(), 3);
841 assert_eq!(prepared.run_at(), Some(run_at));
842 assert_eq!(prepared.metadata()["source"], "test");
843 assert_eq!(prepared.tags(), ["welcome", "external"]);
844 assert!(prepared.unique_key().is_some());
845 assert_eq!(prepared.unique_states_bit_string(), Some("10000001"));
846 assert_eq!(prepared.ordering_key(), Some(&[1, 2, 3][..]));
847 }
848
849 #[test]
850 fn prepare_raw_job_insert_reuses_null_byte_validation() {
851 let err = prepare_raw_job_insert(
852 "send_email",
853 serde_json::json!({"subject": "hello\u{0000}world"}),
854 InsertOpts::default(),
855 )
856 .unwrap_err();
857
858 assert!(matches!(err, AwaError::Validation(_)));
859 }
860
861 #[test]
862 fn postgres_adapter_sql_uses_driver_friendly_casts() {
863 assert!(POSTGRES_INSERT_JOB_SQL.contains("state::text AS state_str"));
864 assert!(POSTGRES_INSERT_JOB_SQL.contains("$4::text::awa.job_state"));
865 assert!(POSTGRES_INSERT_JOB_SQL.contains("$11::text::bit(8)"));
866 assert!(POSTGRES_INSERT_JOB_SQL.contains("$12"));
867 }
868}