use super::*;
impl ValiseFile {
pub fn create_collection(&mut self, name: impl Into<String>) -> Result<CollectionId> {
self.create_collection_at(name, current_unix_timestamp()?)
}
pub fn create_collection_at(
&mut self,
name: impl Into<String>,
created_at: i64,
) -> Result<CollectionId> {
self.ensure_write()?;
let name = name.into();
if self
.catalog
.collections
.iter()
.any(|collection| collection.name == name && collection.flags & 1 == 0)
{
return Err(Error::Format(format!("collection already exists: {name}")));
}
let collection_id = self.id_allocator.allocate_collection_id()?;
let collection = CollectionDesc {
collection_id,
name,
created_at,
default_text_space_id: None,
default_embedding_space_id: None,
flags: 0,
metadata_ref: None,
};
upsert_collection(&mut self.catalog, collection);
self.dirty_collection_ids.insert(collection_id);
self.dirty = true;
Ok(collection_id)
}
pub fn put_frame(&mut self, input: PutFrame<'_>) -> Result<FrameId> {
self.ensure_write()?;
self.ensure_segment_registry_loaded()?;
let prof = self.ingest_profile_enabled;
let _call_start = std::time::Instant::now();
let _t = std::time::Instant::now();
if !self
.catalog
.collections
.iter()
.any(|collection| collection.collection_id == input.collection_id)
{
return Err(Error::Format(format!(
"unknown collection id: {}",
input.collection_id.0
)));
}
if prof {
let p = &self.ingest_profile;
p.put_frame_collection_lookup_ns.fetch_add(
_t.elapsed().as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
let _t = std::time::Instant::now();
let now = current_unix_timestamp()?;
let frame_id = self.id_allocator.allocate_frame_id()?;
if prof {
let p = &self.ingest_profile;
p.put_frame_allocate_ids_ns.fetch_add(
_t.elapsed().as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
let _t = std::time::Instant::now();
let bytes_offset = self.pending_payload_buf.len() as u64;
let bytes_length = input.payload.len() as u64;
self.pending_payload_buf.extend_from_slice(input.payload);
let pending_index = self.pending_payload_frames.len();
self.pending_payload_frames.push(PendingPayloadFrame {
frame_id,
bytes_offset,
bytes_length,
});
self.pending_payload_by_frame
.insert(frame_id, pending_index);
if let Some(uri) = input.uri {
let uri_offset = self.pending_uri_buf.len() as u64;
self.pending_uri_buf.extend_from_slice(uri);
self.pending_uris.push(PendingUri {
frame_id,
bytes_offset: uri_offset,
bytes_length: uri.len() as u64,
});
}
let frame = FrameDesc {
frame_id,
collection_id: input.collection_id,
role: input.role,
created_at: input.created_at.unwrap_or(now),
updated_at: input.updated_at.unwrap_or(now),
uri_ref: None,
title_ref: None,
payload_ref: None,
payload_encoding: PayloadEncoding::Raw,
search_text_ref: None,
parent_frame_id: input.parent_frame_id,
chunk_index: input.chunk_index,
chunk_count: input.chunk_count,
metadata_ref: None,
status: FrameStatus::Active,
};
if prof {
let p = &self.ingest_profile;
p.put_frame_segment_write_ns.fetch_add(
_t.elapsed().as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
let _t = std::time::Instant::now();
upsert_frame(&mut self.catalog, frame);
self.dirty_frame_ids.insert(frame_id);
self.dirty = true;
if prof {
let p = &self.ingest_profile;
p.put_frame_catalog_upsert_ns.fetch_add(
_t.elapsed().as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
if self.pending_payload_buf.len() >= PAYLOAD_BATCH_FLUSH_BYTES {
self.flush_pending_payloads()?;
}
if prof {
let p = &self.ingest_profile;
p.put_frame_total_ns.fetch_add(
_call_start.elapsed().as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
p.put_frame_calls
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
Ok(frame_id)
}
pub(crate) fn flush_pending_payloads(&mut self) -> Result<()> {
if self.pending_payload_frames.is_empty() {
debug_assert!(self.pending_payload_buf.is_empty());
return Ok(());
}
let segment_id = self.id_allocator.allocate_segment_id()?;
let item_count = u32::try_from(self.pending_payload_frames.len())
.map_err(|_| Error::Format("payload batch too large for u32 item_count".into()))?;
let uncompressed_length = self.pending_payload_buf.len() as u64;
let compressed = zstd::encode_all(self.pending_payload_buf.as_slice(), 3)
.map_err(|err| Error::Format(format!("zstd encode failed: {err}")))?;
let payload_segment = crate::file::segment_io::append_compressed_segment_at_end(
&mut self.file.lock(),
segment_id,
SegmentType::Payload,
item_count,
&compressed,
uncompressed_length,
crate::format::segment::Compression::Zstd,
)?;
sync_file(&self.file.lock(), self.durability)?;
register_segment(
SegmentRegistryMut {
registry: self.segment_registry.get_mut(),
dirty_ids: &mut self.dirty_segment_ids,
},
crate::file::segment_io::compressed_segment_catalog_entry(
segment_id,
SegmentType::Payload,
item_count,
&compressed,
uncompressed_length,
crate::format::segment::Compression::Zstd,
payload_segment,
)?,
);
let uri_refs_by_frame: HashMap<FrameId, crate::format::catalog::MetadataRef> =
if self.pending_uris.is_empty() {
HashMap::new()
} else {
let uri_segment_id = self.id_allocator.allocate_segment_id()?;
let uri_item_count = u32::try_from(self.pending_uris.len())
.map_err(|_| Error::Format("uri batch too large for u32 item_count".into()))?;
let uri_segment = append_segment_at_end(
&mut self.file.lock(),
uri_segment_id,
SegmentType::Metadata,
uri_item_count,
&self.pending_uri_buf,
)?;
sync_file(&self.file.lock(), self.durability)?;
register_segment(
SegmentRegistryMut {
registry: self.segment_registry.get_mut(),
dirty_ids: &mut self.dirty_segment_ids,
},
segment_catalog_entry(
uri_segment_id,
SegmentType::Metadata,
uri_item_count,
&self.pending_uri_buf,
uri_segment,
)?,
);
let pending_uris = std::mem::take(&mut self.pending_uris);
let mut map = HashMap::with_capacity(pending_uris.len());
for u in &pending_uris {
let start = u.bytes_offset as usize;
let end = start + u.bytes_length as usize;
let checksum =
crate::format::blake3_checksum(&self.pending_uri_buf[start..end]);
map.insert(
u.frame_id,
crate::format::catalog::MetadataRef {
segment_id: uri_segment_id,
bytes_offset: u.bytes_offset,
bytes_length: u.bytes_length,
checksum,
},
);
self.dirty_frame_ids.insert(u.frame_id);
}
map
};
let pending = std::mem::take(&mut self.pending_payload_frames);
let mut refs_by_frame: HashMap<FrameId, (u64, u64)> = HashMap::with_capacity(pending.len());
for entry in &pending {
refs_by_frame.insert(entry.frame_id, (entry.bytes_offset, entry.bytes_length));
self.dirty_frame_ids.insert(entry.frame_id);
}
let mut matched = 0usize;
for frame in &mut self.catalog.frames {
if let Some(&(offset, length)) = refs_by_frame.get(&frame.frame_id) {
frame.payload_ref = Some(PayloadRef {
segment_id,
bytes_offset: offset,
bytes_length: length,
});
matched += 1;
}
if let Some(uri_ref) = uri_refs_by_frame.get(&frame.frame_id) {
frame.uri_ref = Some(uri_ref.clone());
}
}
if matched != refs_by_frame.len() {
return Err(Error::Integrity(format!(
"flush_pending_payloads: matched {matched}/{} pending frames in catalog",
refs_by_frame.len()
)));
}
self.pending_payload_buf.clear();
self.pending_payload_by_frame.clear();
self.pending_uri_buf.clear();
Ok(())
}
}