1use std::collections::BTreeMap;
10use std::sync::Arc;
11use uqa_core::{DocId, Value};
12use uqa_storage::read_control::StorageReadControl;
13use uqa_storage::{
14 document_store::Document, DocumentMetadata, DocumentStore, RetainedDocumentFields,
15 StorageBackendError, StorageBackendResult, StoredDocument,
16};
17
18mod visible;
19pub use visible::VisibleDocumentIds;
20mod desired;
21mod identities;
22mod layers;
23mod projection;
24pub use desired::DocumentSelection;
25pub use identities::{
26 ChangedIdentities, ChangedIdentitiesView, ChangedRowCounts, RelationGeneration,
27};
28pub use layers::Changes;
29use layers::{ChangeRef, IdentityLayer, StagedLayers};
30mod selection;
31use selection::Selection;
32mod vectors;
33use vectors::CapturedRows;
34
35#[cfg(test)]
36mod tests;
37
38#[derive(Clone)]
39enum Change {
40 Deleted,
41 Fields(RetainedDocumentFields, DocumentMetadata),
42 Retained(Arc<dyn DocumentStore>),
43 Captured(Arc<uqa_core::memory::Budgeted<CapturedRows>>, bool),
44}
45
46impl Change {
47 fn present(&self) -> bool {
48 !matches!(self, Self::Deleted | Self::Captured(_, false))
49 }
50
51 fn fields(&self) -> Option<&Document> {
52 match self {
53 Self::Fields(fields, _) => Some(fields),
54 Self::Deleted | Self::Retained(_) | Self::Captured(_, _) => None,
55 }
56 }
57
58 fn into_stored(self, id: DocId) -> StorageBackendResult<Option<StoredDocument>> {
59 Ok(match self {
60 Self::Deleted | Self::Captured(_, false) => None,
61 Self::Fields(fields, metadata) => Some(StoredDocument::with_metadata(
62 fields.into_document(),
63 metadata,
64 )),
65 Self::Retained(source) => return source.get_stored(id),
66 Self::Captured(source, true) => return source.documents.get_stored(id),
67 })
68 }
69
70 fn retained_source(&self) -> Option<&Arc<dyn DocumentStore>> {
71 match self {
72 Self::Retained(source) => Some(source),
73 Self::Captured(source, true) => Some(&source.documents),
74 Self::Deleted | Self::Fields(_, _) | Self::Captured(_, false) => None,
75 }
76 }
77}
78
79#[derive(Clone, Default)]
81pub struct DocumentChanges {
82 selection: Option<Arc<Selection>>,
83 identities: Option<Arc<IdentityLayer>>,
85 staged: Option<Arc<StagedLayers>>,
87}
88
89impl DocumentChanges {
90 pub(crate) fn without_staged(&self) -> Self {
92 Self {
93 selection: self.selection.clone(),
94 identities: self.identities.clone(),
95 staged: None,
96 }
97 }
98
99 pub fn capture_owned(
101 source: &dyn DocumentStore,
102 desired: DocumentSelection,
103 control: &StorageReadControl,
104 ) -> StorageBackendResult<Self> {
105 let mut result = Self::default();
106 let desired = desired.finish(control)?;
107 let mut desired = desired.entries();
108 loop {
109 control.check()?;
110 let mut page = uqa_core::memory::BudgetedVec::new(control.memory());
111 for row in desired.by_ref().take(crate::DEFAULT_BATCH_SIZE) {
112 page.push(row)?;
113 }
114 if page.is_empty() {
115 break;
116 }
117 let mut ids = uqa_core::memory::BudgetedVec::new(control.memory());
118 for (id, present) in page.iter() {
119 if *present {
120 ids.push(*id)?;
121 }
122 }
123 let (documents, _page_memory) =
124 uqa_storage::document_store::read_stored_documents(source, &ids, control)?
125 .into_parts();
126 let mut documents = documents.into_iter();
127 for (id, present) in page.iter().copied() {
128 control.check()?;
129 let row = if present {
130 documents.next().expect("validated whole-row page length")
131 } else {
132 None
133 };
134 let change = row.map_or(Change::Deleted, |row| {
135 let (fields, metadata) = row.into_parts();
136 Change::Fields(fields, metadata)
137 });
138 result.insert(id, change, control)?;
139 }
140 }
141 Ok(result)
142 }
143
144 pub fn has_changes(&self) -> bool {
145 !self.rows().is_empty() || self.identities.is_some() || !self.staged_views().is_empty()
146 }
147
148 pub fn contains_change(&self, id: DocId) -> StorageBackendResult<bool> {
149 Ok(self.layers_change(id)? || self.selected(id).is_some())
150 }
151
152 pub fn change_presence(&self, id: DocId) -> StorageBackendResult<Option<bool>> {
153 Ok(self.get(id)?.map(|change| change.present()))
154 }
155
156 pub fn changes(&self) -> Changes<bool, bool> {
158 self.changes_after(None)
159 }
160
161 pub fn with_retained(
163 mut self,
164 source: Arc<dyn DocumentStore>,
165 desired: DocumentSelection,
166 control: &StorageReadControl,
167 ) -> StorageBackendResult<Self> {
168 control.check()?;
169 let desired = desired.finish(control)?;
170 let mut newer = Self::default();
171 for (id, present) in desired.entries() {
172 control.check()?;
173 let change = if present && source.contains_doc_id(id)? {
174 Change::Retained(Arc::clone(&source))
175 } else {
176 Change::Deleted
177 };
178 newer.insert(id, change, control)?;
179 }
180 self.extend(newer, control)?;
181 Ok(self)
182 }
183
184 pub fn insert_shared(
185 &mut self,
186 id: DocId,
187 document: Option<(Arc<Document>, DocumentMetadata)>,
188 control: &StorageReadControl,
189 ) -> StorageBackendResult<()> {
190 let control = self.retention_control(control);
191 let document = document
192 .map(|(fields, metadata)| {
193 RetainedDocumentFields::new(fields, &control).map(|fields| (fields, metadata))
194 })
195 .transpose()?;
196 self.insert(
197 id,
198 document.map_or(Change::Deleted, |(fields, metadata)| {
199 Change::Fields(fields, metadata)
200 }),
201 &control,
202 )
203 }
204
205 fn insert_owned(
206 &mut self,
207 id: DocId,
208 row: Option<StoredDocument>,
209 control: &StorageReadControl,
210 ) -> StorageBackendResult<()> {
211 self.insert_shared(
212 id,
213 row.map(|row| {
214 let (fields, metadata) = row.into_parts();
215 (Arc::new(fields), metadata)
216 }),
217 control,
218 )
219 }
220
221 pub fn from_retained(
223 rows: impl IntoIterator<Item = (DocId, Option<(RetainedDocumentFields, DocumentMetadata)>)>,
224 control: &StorageReadControl,
225 ) -> StorageBackendResult<Self> {
226 control.check()?;
227 let mut selected = Self::default();
228 for (id, row) in rows {
229 selected.insert(
230 id,
231 row.map_or(Change::Deleted, |(fields, metadata)| {
232 Change::Fields(fields, metadata)
233 }),
234 control,
235 )?;
236 }
237 control.check()?;
238 Ok(selected)
239 }
240
241 pub fn from_shared(
243 rows: impl IntoIterator<Item = (DocId, Option<(Arc<Document>, DocumentMetadata)>)>,
244 control: &StorageReadControl,
245 ) -> StorageBackendResult<Self> {
246 control.check()?;
247 let mut selected = Self::default();
248 for (id, row) in rows {
249 selected.insert_shared(id, row, control)?;
250 }
251 control.check()?;
252 Ok(selected)
253 }
254
255 pub fn from_rows(
256 rows: BTreeMap<DocId, Option<StoredDocument>>,
257 control: &StorageReadControl,
258 ) -> StorageBackendResult<Self> {
259 control.check()?;
260 let mut selected = Self::default();
261 for (id, row) in rows {
262 selected.insert_owned(id, row, control)?;
263 }
264 control.check()?;
265 Ok(selected)
266 }
267
268 pub fn into_rows(
269 self,
270 ) -> impl Iterator<Item = StorageBackendResult<(DocId, Option<StoredDocument>)>> {
271 if self.staged.is_some() || self.identities.is_some() {
272 return Box::new(self.change_rows_after(None).map(|change| {
274 change.and_then(|(id, change)| change.into_stored(id).map(|row| (id, row)))
275 }))
276 as Box<dyn Iterator<Item = StorageBackendResult<(DocId, Option<StoredDocument>)>>>;
277 }
278 let (mut owned, shared) = match self.selection.map(Arc::try_unwrap) {
279 Some(Ok(selection)) => {
280 let (rows, capacity, allocation) = selection.into_parts();
281 (
282 Some((rows.into_iter(), capacity, allocation)),
283 Self::default(),
284 )
285 }
286 Some(Err(selection)) => (
287 None,
288 Self {
289 selection: Some(selection),
290 identities: None,
291 staged: None,
292 },
293 ),
294 None => (None, Self::default()),
295 };
296 let mut position = 0;
297 Box::new(std::iter::from_fn(move || {
298 let (id, change) = if let Some((rows, _, _)) = owned.as_mut() {
299 rows.next()?
300 } else {
301 let row = shared.rows().get(position)?.clone();
302 position += 1;
303 row
304 };
305 Some(change.into_stored(id).map(|row| (id, row)))
306 }))
307 }
308}
309
310fn readonly() -> StorageBackendError {
311 StorageBackendError::Other("captured document changes are read-only".into())
312}
313
314impl DocumentStore for DocumentChanges {
315 fn put(&mut self, _: DocId, _: Document) -> StorageBackendResult<()> {
316 Err(readonly())
317 }
318
319 fn put_stored(&mut self, _: DocId, _: StoredDocument) -> StorageBackendResult<()> {
320 Err(readonly())
321 }
322
323 fn patch_fields(
324 &mut self,
325 _: DocId,
326 _: &BTreeMap<String, Value>,
327 ) -> StorageBackendResult<bool> {
328 Err(readonly())
329 }
330
331 fn delete(&mut self, _: DocId) -> StorageBackendResult<()> {
332 Err(readonly())
333 }
334
335 fn clear(&mut self) -> StorageBackendResult<()> {
336 Err(readonly())
337 }
338
339 fn get_stored(&self, id: DocId) -> StorageBackendResult<Option<StoredDocument>> {
340 self.get(id)?
341 .map_or(Change::Deleted, ChangeRef::into_owned)
342 .into_stored(id)
343 }
344
345 fn get_stored_many(
346 &self,
347 ids: &[DocId],
348 ) -> StorageBackendResult<BTreeMap<DocId, StoredDocument>> {
349 let mut rows = BTreeMap::new();
350 let mut staged = self.layer_reader(ids);
351 let mut index = 0;
352 while index < ids.len() {
353 if let Some((end, source)) = self.batch_source_run(ids, index, &mut staged)? {
354 rows.extend(source.get_stored_many(&ids[index..end])?);
355 index = end;
356 } else {
357 let id = ids[index];
358 let change = self
359 .batch_change(&mut staged, id)?
360 .map_or(Change::Deleted, ChangeRef::into_owned);
361 if let Some(row) = change.into_stored(id)? {
362 rows.insert(id, row);
363 }
364 index += 1;
365 }
366 }
367 Ok(rows)
368 }
369
370 fn get_stored_many_controlled(
371 &self,
372 ids: &[DocId],
373 control: &StorageReadControl,
374 ) -> StorageBackendResult<uqa_storage::RetainedDocumentPage> {
375 use uqa_storage::{document_store::read_stored_documents, RetainedStoredDocument};
376 control.check()?;
377 let mut rows = uqa_core::memory::BudgetedVec::new(control.memory());
378 rows.reserve(ids.len())?;
379 let mut staged = self.layer_reader(ids);
380 let mut index = 0;
381 while index < ids.len() {
382 control.check()?;
383 if let Some((end, source)) = self.batch_source_run(ids, index, &mut staged)? {
384 let (page, _memory) =
385 read_stored_documents(source.as_ref(), &ids[index..end], control)?.into_parts();
386 for row in page {
387 control.check()?;
388 rows.push(row)?;
389 }
390 index = end;
391 } else {
392 let change = self.batch_change(&mut staged, ids[index])?;
393 let row = match change.as_deref() {
394 Some(Change::Fields(fields, metadata)) => {
395 Some(RetainedStoredDocument::with_metadata(
396 fields.retain_with_control(control)?,
397 *metadata,
398 ))
399 }
400 Some(Change::Deleted | Change::Captured(_, false)) | None => None,
401 Some(Change::Retained(_) | Change::Captured(_, true)) => {
402 unreachable!("retained source run")
403 }
404 };
405 rows.push(row)?;
406 index += 1;
407 }
408 }
409 control.check()?;
410 Ok(rows)
411 }
412
413 fn get_metadata(&self, id: DocId) -> StorageBackendResult<Option<DocumentMetadata>> {
414 Ok(match self.get(id)?.as_deref() {
415 Some(Change::Fields(_, metadata)) => Some(*metadata),
416 Some(Change::Retained(source)) => return source.get_metadata(id),
417 Some(Change::Captured(source, true)) => return source.documents.get_metadata(id),
418 Some(Change::Deleted | Change::Captured(_, false)) | None => None,
419 })
420 }
421
422 fn with_field_ref_controlled(
423 &self,
424 id: DocId,
425 field: &str,
426 control: &StorageReadControl,
427 visitor: &mut dyn FnMut(Option<&Value>) -> StorageBackendResult<()>,
428 ) -> StorageBackendResult<()> {
429 control.check()?;
430 let change = self.get(id)?;
431 if let Some(source) = change.as_deref().and_then(Change::retained_source) {
432 source.with_field_ref_controlled(id, field, control, visitor)?;
433 } else {
434 visitor(
435 change
436 .as_deref()
437 .and_then(Change::fields)
438 .and_then(|fields| fields.get(field)),
439 )?;
440 }
441 control.check()
442 }
443
444 fn field_presence_controlled(
445 &self,
446 ids: &[DocId],
447 fields: &[&str],
448 control: &StorageReadControl,
449 ) -> StorageBackendResult<uqa_core::memory::BudgetedVec<bool>> {
450 control.check()?;
451 let mut present = uqa_core::memory::BudgetedVec::new(control.memory());
452 if fields.is_empty() {
453 return Ok(present);
454 }
455 let mut staged = self.layer_reader(ids);
456 let mut index = 0;
457 while index < ids.len() {
458 control.check()?;
459 if let Some((end, source)) = self.batch_source_run(ids, index, &mut staged)? {
460 let page = uqa_storage::document_store::read_field_presence(
461 source.as_ref(),
462 &ids[index..end],
463 fields,
464 control,
465 )?;
466 present.extend_from_slice(&page)?;
467 index = end;
468 } else {
469 let change = self.batch_change(&mut staged, ids[index])?;
470 let row = change.as_deref().and_then(Change::fields);
471 for field in fields {
472 control.check()?;
473 present.push(row.is_some_and(|row| row.contains_key(*field)))?;
474 }
475 index += 1;
476 }
477 }
478 control.check()?;
479 Ok(present)
480 }
481
482 fn contains_doc_id(&self, id: DocId) -> StorageBackendResult<bool> {
483 Ok(self.change_presence(id)? == Some(true))
484 }
485
486 fn get_field(&self, id: DocId, field: &str) -> StorageBackendResult<Option<Value>> {
487 let change = self.get(id)?;
488 match change.as_deref().and_then(Change::retained_source) {
489 Some(source) => source.get_field(id, field),
490 None => Ok(change
491 .as_deref()
492 .and_then(Change::fields)
493 .and_then(|fields| fields.get(field))
494 .cloned()),
495 }
496 }
497
498 fn get_fields_multi(
499 &self,
500 ids: &[DocId],
501 fields: &[&str],
502 ) -> StorageBackendResult<BTreeMap<DocId, Vec<Value>>> {
503 let mut rows = BTreeMap::new();
504 self.for_each_fields_multi_ref_with_presence(ids, fields, &mut |id, present, values| {
505 if present {
506 rows.insert(id, values.iter().map(|value| (*value).clone()).collect());
507 }
508 true
509 })?;
510 Ok(rows)
511 }
512
513 fn for_each_fields_multi_ref_with_presence(
514 &self,
515 ids: &[DocId],
516 fields: &[&str],
517 visitor: &mut dyn FnMut(DocId, bool, &[&Value]) -> bool,
518 ) -> StorageBackendResult<()> {
519 self.visit_projection(ids, fields, visitor)
520 }
521
522 fn for_each_fields_multi_ref(
523 &self,
524 ids: &[DocId],
525 fields: &[&str],
526 visitor: &mut dyn FnMut(DocId, &[&Value]) -> bool,
527 ) -> StorageBackendResult<()> {
528 self.visit_projection(ids, fields, &mut |id, _, values| visitor(id, values))
529 }
530
531 fn for_each_fields_multi(
532 &self,
533 ids: &[DocId],
534 fields: &[&str],
535 visitor: &mut dyn FnMut(DocId, Vec<Value>) -> bool,
536 ) -> StorageBackendResult<()> {
537 self.visit_projection(ids, fields, &mut |id, _, values| {
538 visitor(id, values.iter().map(|value| (*value).clone()).collect())
539 })
540 }
541
542 fn get_shared_fields(
543 &self,
544 ids: &[DocId],
545 fields: &[&str],
546 ) -> StorageBackendResult<Option<Vec<Option<uqa_storage::SharedDocumentRow>>>> {
547 if ids.is_empty() {
548 return Ok(Some(Vec::new()));
549 }
550 match self.batch_source_run(ids, 0, &mut self.layer_reader(ids))? {
551 Some((end, source)) if end == ids.len() => source.get_shared_fields(ids, fields),
552 _ => Ok(None),
553 }
554 }
555
556 fn doc_ids(&self) -> StorageBackendResult<Vec<DocId>> {
557 let mut ids = Vec::new();
558 for change in self.changes() {
559 let (id, present) = change?;
560 if present {
561 ids.push(id);
562 }
563 }
564 Ok(ids)
565 }
566
567 fn next_doc_ids(&self, after: Option<DocId>, limit: usize) -> StorageBackendResult<Vec<DocId>> {
568 let mut ids = Vec::new();
569 for change in self.changes_after(after) {
570 if ids.len() == limit {
571 break;
572 }
573 let (id, present) = change?;
574 if present {
575 ids.push(id);
576 }
577 }
578 Ok(ids)
579 }
580
581 fn next_doc_ids_controlled(
582 &self,
583 after: Option<DocId>,
584 limit: usize,
585 control: &StorageReadControl,
586 ) -> StorageBackendResult<uqa_core::memory::BudgetedVec<DocId>> {
587 control.check()?;
588 let mut ids = uqa_core::memory::BudgetedVec::new(control.memory());
589 if limit == 0 {
590 return Ok(ids);
591 }
592 for change in self.changes_after(after) {
593 control.check()?;
594 if ids.len() == limit {
595 break;
596 }
597 let (id, present) = change?;
598 if present {
599 ids.push(id)?;
600 }
601 }
602 control.check()?;
603 Ok(ids)
604 }
605
606 fn len(&self) -> StorageBackendResult<usize> {
607 let mut count = 0;
608 for change in self.changes() {
609 if change?.1 {
610 count += 1;
611 }
612 }
613 Ok(count)
614 }
615
616 fn snapshot(&self) -> StorageBackendResult<Arc<dyn DocumentStore>> {
617 Ok(Arc::new(self.clone()))
618 }
619}