use std::io::{BufRead, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use chrono::Utc;
use khive_runtime::{KhiveRuntime, RuntimeError};
use khive_storage::types::{SqlStatement, SqlValue};
use khive_storage::SqlWriter;
use khive_types::hash::framed_text_sha256;
use super::parse;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MirrorSource {
ClaudeCode,
Codex,
ChatGptExport,
ClaudeAiExport,
}
impl MirrorSource {
pub fn as_str(self) -> &'static str {
match self {
MirrorSource::ClaudeCode => "claude_code",
MirrorSource::Codex => "codex",
MirrorSource::ChatGptExport => "chatgpt_export",
MirrorSource::ClaudeAiExport => "claude_ai_export",
}
}
}
impl From<LineTailSource> for MirrorSource {
fn from(source: LineTailSource) -> Self {
match source {
LineTailSource::ClaudeCode => MirrorSource::ClaudeCode,
LineTailSource::Codex => MirrorSource::Codex,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LineTailSource {
ClaudeCode,
Codex,
}
#[derive(Debug, Default, Clone, PartialEq)]
pub struct MirrorStats {
pub inserted: u64,
pub replay_mismatches: u64,
pub scanned: u64,
pub new_offset: u64,
pub skipped_oversized_bytes: bool,
pub file_identity: Option<String>,
}
pub(crate) fn file_identity(file: &std::fs::File) -> std::io::Result<String> {
#[cfg(unix)]
{
let identity = khive_fs::fd_relative::FileIdentity::of(file)?;
Ok(format!("unix:{}:{}", identity.dev, identity.ino))
}
#[cfg(windows)]
{
use std::fmt::Write as _;
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Storage::FileSystem::{
FileIdInfo, GetFileInformationByHandleEx, FILE_ID_INFO,
};
let mut info = FILE_ID_INFO::default();
let ok = unsafe {
GetFileInformationByHandleEx(
file.as_raw_handle(),
FileIdInfo,
(&raw mut info).cast(),
std::mem::size_of::<FILE_ID_INFO>() as u32,
)
};
if ok == 0 {
return Err(std::io::Error::last_os_error());
}
let mut identity = format!("windows:{:016x}:", info.VolumeSerialNumber);
for byte in info.FileId.Identifier {
let _ = write!(identity, "{byte:02x}");
}
Ok(identity)
}
#[cfg(not(any(unix, windows)))]
{
Ok(format!("created:{:?}", file.metadata()?.created().ok()))
}
}
fn open_source_file(path: &Path) -> std::io::Result<std::fs::File> {
#[cfg(unix)]
{
use khive_fs::opened_file::{open_regular_file_nofollow, ContainedOpenError};
open_regular_file_nofollow(path).map_err(|error| match error {
ContainedOpenError::Open(error)
| ContainedOpenError::Metadata(error)
| ContainedOpenError::Resolve(error) => error,
ContainedOpenError::NotRegular => std::io::Error::new(
std::io::ErrorKind::InvalidData,
"mirror source is not a regular file",
),
ContainedOpenError::Escapes { .. } => std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"mirror source is outside its configured root",
),
})
}
#[cfg(windows)]
{
windows_source_open::open_file(path)
}
#[cfg(not(any(unix, windows)))]
{
let mut options = std::fs::OpenOptions::new();
options.read(true);
options.open(path)
}
}
#[derive(Clone, Copy)]
pub(crate) struct TrustedSource<'a> {
pub(crate) root: &'a Path,
pub(crate) directory_identities: &'a [String],
}
#[cfg(unix)]
mod root_walk;
#[cfg(unix)]
use root_walk::open_source_root;
#[cfg(unix)]
pub(crate) fn open_source_file_beneath(
root: &Path,
path: &Path,
expected_directories: Option<&[String]>,
) -> std::io::Result<(std::fs::File, Vec<String>)> {
use std::os::unix::ffi::OsStrExt;
use std::path::Component;
let relative = path.strip_prefix(root).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"mirror source is outside its configured root",
)
})?;
let mut components = relative.components().peekable();
if components.peek().is_none() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"mirror source does not name a file beneath its configured root",
));
}
let mut pinned_directories = open_source_root(root)?;
let mut directory_identities = Vec::new();
let root_identity = file_identity(
pinned_directories
.last()
.expect("configured root is retained"),
)?;
if expected_directories.is_some_and(|expected| expected.first() != Some(&root_identity)) {
return Err(std::io::Error::other(
"mirror source root changed after its metadata probe",
));
}
if expected_directories.is_none() {
directory_identities.push(root_identity);
}
let mut directory_depth = 1;
while let Some(component) = components.next() {
let Component::Normal(name) = component else {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"mirror source contains a non-normal path component",
));
};
if name.as_bytes().contains(&0) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"mirror source contains a NUL byte",
));
}
let last = components.peek().is_none();
let directory = pinned_directories
.last()
.expect("source parent is retained");
let opened = khive_fs::fd_relative::open_at(directory, name, !last)?;
if last {
if expected_directories.is_some_and(|expected| expected.len() != directory_depth) {
return Err(std::io::Error::other(
"mirror source ancestor count changed after its metadata probe",
));
}
return Ok((opened, directory_identities));
}
let identity = file_identity(&opened)?;
if expected_directories
.is_some_and(|expected| expected.get(directory_depth) != Some(&identity))
{
return Err(std::io::Error::other(
"mirror source parent changed after its metadata probe",
));
}
if expected_directories.is_none() {
directory_identities.push(identity);
}
directory_depth += 1;
pinned_directories.push(opened);
}
unreachable!("nonempty component iterator must return its final file")
}
#[cfg(windows)]
pub(crate) use windows_source_open::open_source_file_beneath;
#[cfg(windows)]
mod windows_source_open {
use std::ffi::OsStr;
use std::fs::{File, OpenOptions};
use std::io;
use std::os::windows::ffi::OsStrExt;
use std::os::windows::fs::{MetadataExt, OpenOptionsExt};
use std::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle};
use std::path::{Component, Path};
use windows_sys::Wdk::Foundation::OBJECT_ATTRIBUTES;
use windows_sys::Wdk::Storage::FileSystem::{
NtCreateFile, FILE_DIRECTORY_FILE, FILE_NON_DIRECTORY_FILE, FILE_OPEN,
FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
};
use windows_sys::Win32::Foundation::{
RtlNtStatusToDosError, HANDLE, OBJ_CASE_INSENSITIVE, UNICODE_STRING,
};
use windows_sys::Win32::Storage::FileSystem::{
FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS,
FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_READ_DATA, FILE_SHARE_DELETE,
FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_TRAVERSE, SYNCHRONIZE,
};
use windows_sys::Win32::System::IO::IO_STATUS_BLOCK;
use super::file_identity;
fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidInput, message)
}
fn verify_handle(file: &File, directory: bool) -> io::Result<()> {
let metadata = file.metadata()?;
if metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT != 0
|| (directory && !metadata.is_dir())
|| (!directory && !metadata.is_file())
{
return Err(invalid(
"mirror source component has the wrong kind or is a reparse point",
));
}
Ok(())
}
fn open_root(root: &Path) -> io::Result<Vec<File>> {
let root = if root.as_os_str().is_empty() {
Path::new(".")
} else {
root
};
let absolute = if root.is_absolute() {
root.to_path_buf()
} else {
std::env::current_dir()?.join(root)
};
let mut pinned = Vec::new();
for ancestor in absolute.ancestors().collect::<Vec<_>>().into_iter().rev() {
let file = OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
.open(ancestor)?;
verify_handle(&file, true)?;
pinned.push(file);
}
if pinned.is_empty() {
return Err(invalid("mirror source root is empty"));
}
Ok(pinned)
}
pub(super) fn open_file(path: &Path) -> io::Result<File> {
let file = OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(path)?;
verify_handle(&file, false)?;
Ok(file)
}
fn open_child(directory: &File, name: &OsStr, last: bool) -> io::Result<File> {
let mut wide: Vec<u16> = name.encode_wide().collect();
if wide.is_empty()
|| wide.iter().any(|unit| {
*unit == 0
|| *unit == u16::from(b'/')
|| *unit == u16::from(b'\\')
|| *unit == u16::from(b':')
})
{
return Err(invalid(
"mirror source contains an invalid Windows path component",
));
}
let byte_len = wide
.len()
.checked_mul(std::mem::size_of::<u16>())
.and_then(|length| u16::try_from(length).ok())
.ok_or_else(|| invalid("mirror source component is too long"))?;
let unicode_name = UNICODE_STRING {
Length: byte_len,
MaximumLength: byte_len,
Buffer: wide.as_mut_ptr(),
};
let attributes = OBJECT_ATTRIBUTES {
Length: std::mem::size_of::<OBJECT_ATTRIBUTES>() as u32,
RootDirectory: directory.as_raw_handle(),
ObjectName: &raw const unicode_name,
Attributes: OBJ_CASE_INSENSITIVE,
SecurityDescriptor: std::ptr::null(),
SecurityQualityOfService: std::ptr::null(),
};
let mut io_status = IO_STATUS_BLOCK::default();
let mut handle: HANDLE = std::ptr::null_mut();
let desired_access = if last {
FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE
} else {
FILE_TRAVERSE | FILE_READ_ATTRIBUTES | SYNCHRONIZE
};
let create_options = if last {
FILE_NON_DIRECTORY_FILE
} else {
FILE_DIRECTORY_FILE
} | FILE_OPEN_REPARSE_POINT
| FILE_SYNCHRONOUS_IO_NONALERT;
let share_mode = if last {
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE
} else {
FILE_SHARE_READ | FILE_SHARE_WRITE
};
let status = unsafe {
NtCreateFile(
&raw mut handle,
desired_access,
&raw const attributes,
&raw mut io_status,
std::ptr::null(),
FILE_ATTRIBUTE_NORMAL,
share_mode,
FILE_OPEN,
create_options,
std::ptr::null(),
0,
)
};
if status < 0 {
let error = unsafe { RtlNtStatusToDosError(status) };
return Err(io::Error::from_raw_os_error(error as i32));
}
let file = unsafe { File::from_raw_handle(handle as RawHandle) };
verify_handle(&file, !last)?;
Ok(file)
}
pub(crate) fn open_source_file_beneath(
root: &Path,
path: &Path,
expected_directories: Option<&[String]>,
) -> io::Result<(File, Vec<String>)> {
let relative = path
.strip_prefix(root)
.map_err(|_| invalid("mirror source is outside its configured root"))?;
let mut components = relative.components().peekable();
if components.peek().is_none() {
return Err(invalid(
"mirror source does not name a file beneath its configured root",
));
}
let mut pinned_directories = open_root(root)?;
let root_identity = file_identity(
pinned_directories
.last()
.ok_or_else(|| invalid("mirror source root is empty"))?,
)?;
if expected_directories.is_some_and(|expected| expected.first() != Some(&root_identity)) {
return Err(io::Error::other(
"mirror source root changed after its metadata probe",
));
}
let mut directory_identities = Vec::new();
if expected_directories.is_none() {
directory_identities.push(root_identity);
}
let mut directory_depth = 1;
while let Some(component) = components.next() {
let Component::Normal(name) = component else {
return Err(invalid(
"mirror source contains a non-normal path component",
));
};
let last = components.peek().is_none();
let opened = open_child(
pinned_directories
.last()
.ok_or_else(|| invalid("mirror source root is empty"))?,
name,
last,
)?;
if last {
if expected_directories.is_some_and(|expected| expected.len() != directory_depth) {
return Err(io::Error::other(
"mirror source ancestor count changed after its metadata probe",
));
}
return Ok((opened, directory_identities));
}
let identity = file_identity(&opened)?;
if expected_directories
.is_some_and(|expected| expected.get(directory_depth) != Some(&identity))
{
return Err(io::Error::other(
"mirror source parent changed after its metadata probe",
));
}
if expected_directories.is_none() {
directory_identities.push(identity);
}
directory_depth += 1;
pinned_directories.push(opened);
}
unreachable!("nonempty component iterator must return its final file")
}
#[cfg(test)]
mod tests {
use std::ffi::OsStr;
use super::{open_child, open_root};
use tempfile::TempDir;
#[test]
fn mirror_windows_pinned_intermediate_directory_cannot_leave_root() {
let temp = TempDir::new().expect("tempdir");
let root = temp.path().join("root");
let outside = temp.path().join("outside");
let parent = root.join("staged");
let moved = outside.join("staged");
std::fs::create_dir_all(&parent).expect("staged directory");
std::fs::create_dir_all(&outside).expect("outside directory");
std::fs::write(parent.join("source.jsonl"), b"inside\n").expect("source file");
std::fs::rename(&parent, &moved).expect("unheld directory can leave root");
std::fs::rename(&moved, &parent).expect("restore source directory");
let directories = open_root(&root).expect("pin root ancestors");
let directory = open_child(
directories.last().expect("root handle"),
OsStr::new("staged"),
false,
)
.expect("pin intermediate directory");
assert!(
std::fs::rename(&parent, &moved).is_err(),
"an opened intermediate directory must not leave its configured root"
);
assert!(open_child(&directory, OsStr::new("source.jsonl"), true).is_ok());
drop(directory);
std::fs::rename(&parent, &moved).expect("rename succeeds after releasing directory");
}
}
}
#[cfg(not(any(unix, windows)))]
pub(crate) fn open_source_file_beneath(
_root: &Path,
_path: &Path,
_expected_directories: Option<&[String]>,
) -> std::io::Result<(std::fs::File, Vec<String>)> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"secure directory-relative mirror source opens are unavailable on this platform",
))
}
fn checked_identity(
file: &std::fs::File,
metadata: &std::fs::Metadata,
expected_identity: Option<&str>,
) -> std::io::Result<String> {
if !metadata.is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"mirror source is not a regular file",
));
}
let identity = file_identity(file)?;
if expected_identity.is_some_and(|expected| expected != identity.as_str()) {
return Err(std::io::Error::other(
"mirror source was replaced after its metadata probe",
));
}
Ok(identity)
}
const MIRROR_MAX_BYTES_PER_PASS: usize = 8 * 1024 * 1024;
const MIRROR_MAX_EVENTS_PER_PASS: usize = 1024;
const MIRROR_MAX_LINE_BYTES: usize = MIRROR_MAX_BYTES_PER_PASS;
#[derive(Clone, Copy, Debug)]
struct MirrorLimits {
max_bytes_per_pass: usize,
max_events_per_pass: usize,
max_line_bytes: usize,
}
impl MirrorLimits {
fn production() -> Self {
Self {
max_bytes_per_pass: MIRROR_MAX_BYTES_PER_PASS,
max_events_per_pass: MIRROR_MAX_EVENTS_PER_PASS,
max_line_bytes: MIRROR_MAX_LINE_BYTES,
}
}
}
pub async fn mirror_file(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
) -> Result<MirrorStats, RuntimeError> {
mirror_file_with_limits(
runtime,
path,
start_offset,
source,
codex_session_id,
MirrorLimits::production(),
)
.await
}
pub async fn mirror_file_deferred(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
) -> Result<MirrorStats, RuntimeError> {
mirror_file_deferred_checked(
runtime,
path,
start_offset,
source,
codex_session_id,
None,
None,
)
.await
}
pub(crate) async fn mirror_file_deferred_checked(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
expected_identity: Option<&str>,
trusted_source: Option<TrustedSource<'_>>,
) -> Result<MirrorStats, RuntimeError> {
mirror_file_inner(
runtime,
path,
start_offset,
source,
codex_session_id,
MirrorLimits::production(),
false,
expected_identity,
trusted_source,
)
.await
}
pub async fn commit_empty_advance(
runtime: &KhiveRuntime,
path: &Path,
stats: &MirrorStats,
) -> Result<(), RuntimeError> {
write_cursor_only(
runtime,
path,
&None,
stats.new_offset,
stats.file_identity.as_deref(),
)
.await
}
pub async fn adopt_cursor_identity(
runtime: &KhiveRuntime,
path: &Path,
offset: u64,
identity: &str,
) -> Result<(), RuntimeError> {
write_cursor_only(runtime, path, &None, offset, Some(identity)).await
}
struct MirrorChunk {
events: Vec<parse::ParsedEvent>,
scanned: u64,
new_offset: u64,
skipped_oversized_bytes: bool,
file_identity: String,
}
#[derive(Debug)]
enum LineRead {
Eof,
Partial,
Complete { bytes: usize },
Oversized { bytes: usize },
OversizedUnterminated { bytes: usize },
}
fn read_line_bounded(
reader: &mut impl BufRead,
buf: &mut Vec<u8>,
max_line_bytes: usize,
already_oversized: bool,
) -> std::io::Result<LineRead> {
let mut total: usize = 0;
let mut oversized = already_oversized;
loop {
let available = reader.fill_buf()?;
if available.is_empty() {
return Ok(if total == 0 {
LineRead::Eof
} else {
LineRead::Partial
});
}
let newline_pos = available.iter().position(|&b| b == b'\n');
let take = newline_pos.map_or(available.len(), |pos| pos + 1);
if !oversized {
if total + take > max_line_bytes {
oversized = true;
} else {
buf.extend_from_slice(&available[..take]);
}
}
total += take;
reader.consume(take);
if newline_pos.is_some() {
return Ok(if oversized {
LineRead::Oversized { bytes: total }
} else {
LineRead::Complete { bytes: total }
});
}
if oversized && total > max_line_bytes {
return Ok(LineRead::OversizedUnterminated { bytes: total });
}
}
}
fn read_bounded_chunk(
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
limits: MirrorLimits,
expected_identity: Option<&str>,
trusted_source: Option<TrustedSource<'_>>,
) -> std::io::Result<MirrorChunk> {
let mut file = match trusted_source {
Some(source) => {
open_source_file_beneath(source.root, path, Some(source.directory_identities))?.0
}
None => open_source_file(path)?,
};
let metadata = file.metadata()?;
let identity = checked_identity(&file, &metadata, expected_identity)?;
let file_len = metadata.len();
if start_offset >= file_len {
return Ok(MirrorChunk {
events: Vec::new(),
scanned: 0,
new_offset: start_offset,
skipped_oversized_bytes: false,
file_identity: identity,
});
}
let mut skipping_oversized_line = if start_offset == 0 {
false
} else {
file.seek(SeekFrom::Start(start_offset - 1))?;
let mut previous = [0_u8; 1];
file.read_exact(&mut previous)?;
previous[0] != b'\n'
};
file.seek(SeekFrom::Start(start_offset))?;
let mut reader = std::io::BufReader::new(file);
let mut line = Vec::new();
let mut events = Vec::new();
let mut scanned: u64 = 0;
let mut lines_consumed: u64 = 0;
let mut new_offset = start_offset;
let mut bytes_this_pass: usize = 0;
let mut skipped_oversized_bytes = false;
loop {
if lines_consumed > 0
&& (bytes_this_pass >= limits.max_bytes_per_pass
|| events.len() >= limits.max_events_per_pass)
{
break;
}
line.clear();
let line_offset = new_offset;
match read_line_bounded(
&mut reader,
&mut line,
limits.max_line_bytes,
skipping_oversized_line,
)? {
LineRead::Eof => break,
LineRead::Partial => break, LineRead::OversizedUnterminated { bytes } => {
new_offset += bytes as u64;
skipped_oversized_bytes = true;
tracing::warn!(
path = %path.display(),
offset = line_offset,
next_offset = new_offset,
line_bytes = bytes,
max_line_bytes = limits.max_line_bytes,
"session mirror: oversized JSONL line has no terminator in this bounded read; \
advancing the cursor to continue the bounded skip"
);
break;
}
LineRead::Oversized { bytes } => {
tracing::warn!(
path = %path.display(),
offset = line_offset,
line_bytes = bytes,
max_line_bytes = limits.max_line_bytes,
"session mirror: skipping oversized JSONL line"
);
new_offset += bytes as u64;
bytes_this_pass += bytes;
lines_consumed += 1;
skipping_oversized_line = false;
skipped_oversized_bytes = true;
}
LineRead::Complete { bytes } => {
new_offset += bytes as u64;
bytes_this_pass += bytes;
lines_consumed += 1;
let raw = String::from_utf8_lossy(&line[..line.len() - 1]);
if raw.is_empty() {
continue; }
match source {
LineTailSource::ClaudeCode => {
if let Some(ev) = parse::parse_cc_line(&raw) {
events.push(ev);
}
}
LineTailSource::Codex => {
let sid = codex_session_id.unwrap_or("");
if let Some(ev) = parse::parse_codex_line(&raw, sid, line_offset) {
events.push(ev);
}
}
}
scanned += 1;
}
}
}
Ok(MirrorChunk {
events,
scanned,
new_offset,
skipped_oversized_bytes,
file_identity: identity,
})
}
async fn mirror_file_with_limits(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
limits: MirrorLimits,
) -> Result<MirrorStats, RuntimeError> {
mirror_file_inner(
runtime,
path,
start_offset,
source,
codex_session_id,
limits,
true,
None,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn mirror_file_inner(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
source: LineTailSource,
codex_session_id: Option<&str>,
limits: MirrorLimits,
commit_empty_advance: bool,
expected_identity: Option<&str>,
trusted_source: Option<TrustedSource<'_>>,
) -> Result<MirrorStats, RuntimeError> {
let chunk = read_bounded_chunk(
path,
start_offset,
source,
codex_session_id,
limits,
expected_identity,
trusted_source,
)
.map_err(|e| {
RuntimeError::Internal(format!(
"mirror_file: failed to read {:?} at offset {start_offset}: {e}",
path
))
})?;
if chunk.new_offset == start_offset {
return Ok(MirrorStats {
inserted: 0,
replay_mismatches: 0,
scanned: 0,
new_offset: chunk.new_offset,
skipped_oversized_bytes: false,
file_identity: Some(chunk.file_identity),
});
}
if chunk.events.is_empty() {
if commit_empty_advance {
write_cursor_only(
runtime,
path,
&None,
chunk.new_offset,
Some(&chunk.file_identity),
)
.await?;
}
return Ok(MirrorStats {
inserted: 0,
replay_mismatches: 0,
scanned: chunk.scanned,
new_offset: chunk.new_offset,
skipped_oversized_bytes: chunk.skipped_oversized_bytes,
file_identity: Some(chunk.file_identity),
});
}
write_events_and_cursor(
runtime,
path,
MirrorSource::from(source).as_str(),
&[],
&chunk.events,
chunk.scanned,
chunk.new_offset,
&chunk.file_identity,
)
.await
}
const DEFAULT_CHATGPT_MAX_BYTES: u64 = 256 * 1024 * 1024;
const DEFAULT_CLAUDE_AI_MAX_BYTES: u64 = 256 * 1024 * 1024;
#[derive(Clone, Copy)]
struct WholeFileExportSpec {
source: MirrorSource,
parser: fn(&str) -> Option<parse::ParsedExport>,
operation: &'static str,
format_name: &'static str,
max_bytes_env: &'static str,
}
fn parse_chatgpt_export_for_ingest(content: &str) -> Option<parse::ParsedExport> {
parse::parse_chatgpt_export(content).map(|events| parse::ParsedExport {
sessions: Vec::new(),
events,
})
}
const CHATGPT_EXPORT_SPEC: WholeFileExportSpec = WholeFileExportSpec {
source: MirrorSource::ChatGptExport,
parser: parse_chatgpt_export_for_ingest,
operation: "mirror_chatgpt_export_file",
format_name: "ChatGPT",
max_bytes_env: "KHIVE_MIRROR_CHATGPT_MAX_BYTES",
};
const CLAUDE_AI_EXPORT_SPEC: WholeFileExportSpec = WholeFileExportSpec {
source: MirrorSource::ClaudeAiExport,
parser: parse::parse_claude_ai_export_with_sessions,
operation: "mirror_claude_ai_export_file",
format_name: "claude.ai",
max_bytes_env: "KHIVE_MIRROR_CLAUDE_AI_MAX_BYTES",
};
fn chatgpt_max_bytes() -> u64 {
export_max_bytes(CHATGPT_EXPORT_SPEC.max_bytes_env, DEFAULT_CHATGPT_MAX_BYTES)
}
fn claude_ai_max_bytes() -> u64 {
export_max_bytes(
CLAUDE_AI_EXPORT_SPEC.max_bytes_env,
DEFAULT_CLAUDE_AI_MAX_BYTES,
)
}
fn export_max_bytes(variable: &str, default: u64) -> u64 {
std::env::var(variable)
.ok()
.and_then(|v| v.parse::<u64>().ok())
.filter(|&n| n > 0)
.unwrap_or(default)
}
pub async fn mirror_chatgpt_export_file(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
) -> Result<MirrorStats, RuntimeError> {
mirror_chatgpt_export_file_with_max_bytes(runtime, path, start_offset, chatgpt_max_bytes())
.await
}
async fn mirror_chatgpt_export_file_with_max_bytes(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
max_bytes: u64,
) -> Result<MirrorStats, RuntimeError> {
mirror_whole_file_export(
runtime,
path,
start_offset,
max_bytes,
CHATGPT_EXPORT_SPEC,
None,
None,
)
.await
}
pub(crate) async fn mirror_chatgpt_export_file_checked(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
expected_identity: &str,
trusted_source: TrustedSource<'_>,
) -> Result<MirrorStats, RuntimeError> {
mirror_whole_file_export(
runtime,
path,
start_offset,
chatgpt_max_bytes(),
CHATGPT_EXPORT_SPEC,
Some(expected_identity),
Some(trusted_source),
)
.await
}
pub async fn mirror_claude_ai_export_file(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
) -> Result<MirrorStats, RuntimeError> {
mirror_claude_ai_export_file_with_max_bytes(runtime, path, start_offset, claude_ai_max_bytes())
.await
}
async fn mirror_claude_ai_export_file_with_max_bytes(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
max_bytes: u64,
) -> Result<MirrorStats, RuntimeError> {
mirror_whole_file_export(
runtime,
path,
start_offset,
max_bytes,
CLAUDE_AI_EXPORT_SPEC,
None,
None,
)
.await
}
pub(crate) async fn mirror_claude_ai_export_file_checked(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
expected_identity: &str,
trusted_source: TrustedSource<'_>,
) -> Result<MirrorStats, RuntimeError> {
mirror_whole_file_export(
runtime,
path,
start_offset,
claude_ai_max_bytes(),
CLAUDE_AI_EXPORT_SPEC,
Some(expected_identity),
Some(trusted_source),
)
.await
}
async fn mirror_whole_file_export(
runtime: &KhiveRuntime,
path: &Path,
start_offset: u64,
max_bytes: u64,
spec: WholeFileExportSpec,
expected_identity: Option<&str>,
trusted_source: Option<TrustedSource<'_>>,
) -> Result<MirrorStats, RuntimeError> {
let file = match trusted_source {
Some(source) => {
open_source_file_beneath(source.root, path, Some(source.directory_identities))
.map(|(file, _)| file)
}
None => open_source_file(path),
}
.map_err(|e| {
RuntimeError::Internal(format!("{}: failed to open {path:?}: {e}", spec.operation))
})?;
let metadata = file.metadata().map_err(|e| {
RuntimeError::Internal(format!("{}: failed to stat {path:?}: {e}", spec.operation))
})?;
let identity = checked_identity(&file, &metadata, expected_identity).map_err(|e| {
RuntimeError::Internal(format!(
"{}: failed to verify {path:?}: {e}",
spec.operation
))
})?;
let file_len = metadata.len();
if file_len <= start_offset {
return Ok(MirrorStats {
inserted: 0,
replay_mismatches: 0,
scanned: 0,
new_offset: start_offset,
skipped_oversized_bytes: false,
file_identity: Some(identity),
});
}
if file_len > max_bytes {
tracing::warn!(
path = %path.display(),
source = spec.source.as_str(),
file_bytes = file_len,
max_bytes,
max_bytes_env = spec.max_bytes_env,
"session mirror: skipping oversized whole-file export"
);
return Ok(MirrorStats {
inserted: 0,
replay_mismatches: 0,
scanned: 0,
new_offset: start_offset,
skipped_oversized_bytes: false,
file_identity: Some(identity),
});
}
let mut bytes = Vec::new();
file.take(max_bytes.saturating_add(1))
.read_to_end(&mut bytes)
.map_err(|e| {
RuntimeError::Internal(format!("{}: failed to read {path:?}: {e}", spec.operation))
})?;
if bytes.len() as u64 != file_len {
return Err(RuntimeError::Internal(format!(
"{}: {path:?} changed size during read; retrying",
spec.operation
)));
}
let content = String::from_utf8(bytes).map_err(|e| {
RuntimeError::Internal(format!(
"{}: invalid UTF-8 in {path:?}: {e}",
spec.operation
))
})?;
let parsed = (spec.parser)(&content).ok_or_else(|| {
RuntimeError::Internal(format!(
"{}: {path:?} is not a valid {} export (expected its conversations.json array shape)",
spec.operation, spec.format_name
))
})?;
let scanned = parsed.events.len() as u64;
write_events_and_cursor(
runtime,
path,
spec.source.as_str(),
&parsed.sessions,
&parsed.events,
scanned,
file_len,
&identity,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn write_events_and_cursor(
runtime: &KhiveRuntime,
path: &Path,
source_value: &'static str,
sessions: &[parse::ParsedSession],
events: &[parse::ParsedEvent],
scanned: u64,
new_offset: u64,
file_identity: &str,
) -> Result<MirrorStats, RuntimeError> {
let now_us = Utc::now().timestamp_micros();
let sql = runtime.sql();
let sessions_owned: Vec<parse::ParsedSession> = sessions.to_vec();
let events_owned: Vec<parse::ParsedEvent> = events.to_vec();
let path_owned: PathBuf = path.to_path_buf();
let identity_owned = file_identity.to_string();
let op: khive_storage::AtomicUnitOp = Box::new(move |writer: &mut dyn SqlWriter| {
Box::pin(async move {
write_events_and_cursor_on_writer(
writer,
&path_owned,
source_value,
&sessions_owned,
&events_owned,
MirrorWriteProgress {
scanned,
new_offset,
now_us,
file_identity: &identity_owned,
},
)
.await
.map(|stats| Box::new(stats) as Box<dyn std::any::Any + Send>)
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"session_mirror_write_events_and_cursor",
e,
)
})
})
});
let boxed = sql
.atomic_unit(op)
.await
.map_err(|e| RuntimeError::Internal(format!("mirror: atomic_unit: {e}")))?;
Ok(*boxed.downcast::<MirrorStats>().unwrap_or_else(|_| {
panic!("atomic_unit op for write_events_and_cursor must return MirrorStats")
}))
}
#[allow(clippy::too_many_arguments)]
async fn ensure_session_on_writer(
writer: &mut dyn SqlWriter,
source_value: &'static str,
session_id: &str,
cwd: Option<&str>,
git_branch: Option<&str>,
slug: Option<&str>,
created_at_micros: i64,
now_us: i64,
) -> khive_storage::types::StorageResult<()> {
let created_at = if created_at_micros != 0 {
created_at_micros
} else {
now_us
};
writer
.execute(SqlStatement {
sql: "INSERT INTO sessions \
(id, provider_session_id, source, cwd, git_branch, slug, \
message_count, first_seen_at, last_seen_at, namespace) \
VALUES(?1, ?1, ?2, ?3, ?4, ?5, 0, ?6, ?6, 'local') \
ON CONFLICT(namespace, source, provider_session_id) DO NOTHING"
.into(),
params: vec![
SqlValue::Text(session_id.to_string()),
SqlValue::Text(source_value.to_string()),
cwd.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
git_branch
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
slug.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Integer(created_at),
],
label: Some("session_mirror_create_session".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: session create",
e,
)
})?;
Ok(())
}
#[derive(Clone, Copy)]
struct MirrorWriteProgress<'a> {
scanned: u64,
new_offset: u64,
now_us: i64,
file_identity: &'a str,
}
async fn write_events_and_cursor_on_writer(
writer: &mut dyn SqlWriter,
path: &Path,
source_value: &'static str,
sessions: &[parse::ParsedSession],
events: &[parse::ParsedEvent],
progress: MirrorWriteProgress<'_>,
) -> khive_storage::types::StorageResult<MirrorStats> {
let MirrorWriteProgress {
scanned,
new_offset,
now_us,
file_identity,
} = progress;
let mut inserted: u64 = 0;
let mut replay_mismatches: u64 = 0;
let mut last_session_id: Option<String> = None;
let mut ensured_session_ids = std::collections::HashSet::new();
for session in sessions {
if !ensured_session_ids.insert(session.session_id.clone()) {
continue;
}
ensure_session_on_writer(
writer,
source_value,
&session.session_id,
session.cwd.as_deref(),
session.git_branch.as_deref(),
session.slug.as_deref(),
session.created_at_micros,
now_us,
)
.await?;
}
for ev in events {
let created_at = if ev.created_at_micros != 0 {
ev.created_at_micros
} else {
now_us
};
let event_hash = framed_text_sha256(ev.text.as_deref(), &ev.raw).to_string();
if ensured_session_ids.insert(ev.session_id.clone()) {
ensure_session_on_writer(
writer,
source_value,
&ev.session_id,
ev.cwd.as_deref(),
ev.git_branch.as_deref(),
ev.slug.as_deref(),
ev.created_at_micros,
now_us,
)
.await?;
}
let affected = writer
.execute(SqlStatement {
sql: "INSERT INTO session_messages \
(id, session_id, seq, parent_uuid, is_sidechain, role, \
msg_type, text, raw, created_at, namespace, source, content_hash) \
VALUES(?1, ?2, \
(SELECT COALESCE(MAX(seq),-1)+1 FROM session_messages \
WHERE namespace='local' AND source=?3 AND session_id=?2), \
?4, ?5, ?6, ?7, ?8, ?9, ?10, 'local', ?3, ?11) \
ON CONFLICT(namespace, source, session_id, id) DO NOTHING"
.into(),
params: vec![
SqlValue::Text(ev.uuid.clone()),
SqlValue::Text(ev.session_id.clone()),
SqlValue::Text(source_value.to_string()),
ev.parent_uuid
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Integer(i64::from(ev.is_sidechain)),
ev.role
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Text(ev.msg_type.clone()),
ev.text
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Text(ev.raw.clone()),
SqlValue::Integer(created_at),
SqlValue::Text(event_hash.clone()),
],
label: Some("session_mirror_insert_message".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: message insert",
e,
)
})?;
if affected == 0 {
let stored_hash = writer
.query_scalar(SqlStatement {
sql: "SELECT content_hash FROM session_messages \
WHERE namespace='local' AND source=?1 AND session_id=?2 AND id=?3"
.into(),
params: vec![
SqlValue::Text(source_value.to_string()),
SqlValue::Text(ev.session_id.clone()),
SqlValue::Text(ev.uuid.clone()),
],
label: Some("session_mirror_replay_hash".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: replay hash lookup",
e,
)
})?;
match stored_hash {
Some(SqlValue::Text(hash)) if hash == event_hash => {}
Some(SqlValue::Text(_)) => replay_mismatches += 1,
_ => {
return Err(khive_storage::StorageError::Conflict {
capability: khive_storage::StorageCapability::Sql,
operation: "mirror: message replay".into(),
message: "duplicate scoped event id has no stored content hash".into(),
});
}
}
}
if affected > 0 {
writer
.execute(SqlStatement {
sql: "UPDATE sessions SET \
last_seen_at=MAX(last_seen_at, ?2), \
cwd=COALESCE(cwd, ?3), \
git_branch=COALESCE(git_branch, ?4), \
slug=COALESCE(slug, ?5) \
WHERE namespace='local' AND source=?6 AND provider_session_id=?1"
.into(),
params: vec![
SqlValue::Text(ev.session_id.clone()),
SqlValue::Integer(created_at),
ev.cwd
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
ev.git_branch
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
ev.slug
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Text(source_value.to_string()),
],
label: Some("session_mirror_touch_session".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: session touch",
e,
)
})?;
}
inserted += affected;
last_session_id = Some(ev.session_id.clone());
}
if inserted > 0 {
let mut seen_sessions: Vec<String> = events
.iter()
.map(|e| e.session_id.clone())
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
seen_sessions.sort();
for sid in &seen_sessions {
writer
.execute(SqlStatement {
sql: "UPDATE sessions SET message_count=\
(SELECT COUNT(*) FROM session_messages \
WHERE namespace='local' AND source=?2 AND session_id=?1) \
WHERE namespace='local' AND source=?2 AND provider_session_id=?1"
.into(),
params: vec![
SqlValue::Text(sid.clone()),
SqlValue::Text(source_value.to_string()),
],
label: Some("session_mirror_refresh_count".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: count refresh",
e,
)
})?;
}
}
upsert_cursor_on_writer(
writer,
path,
last_session_id.as_deref(),
new_offset,
now_us,
Some(file_identity),
)
.await?;
if replay_mismatches > 0 {
tracing::warn!(
source = source_value,
path = %path.display(),
replay_mismatches,
"session mirror retained changed-content replay under an existing scoped event id"
);
}
Ok(MirrorStats {
inserted,
replay_mismatches,
scanned,
new_offset,
skipped_oversized_bytes: false,
file_identity: Some(file_identity.to_string()),
})
}
async fn upsert_cursor_on_writer(
writer: &mut dyn SqlWriter,
path: &Path,
session_id: Option<&str>,
new_offset: u64,
now_us: i64,
file_identity: Option<&str>,
) -> khive_storage::types::StorageResult<()> {
let path_str = path.to_string_lossy().into_owned();
writer
.execute(SqlStatement {
sql:
"INSERT INTO session_mirror_cursor(file_path, session_id, byte_offset, updated_at, file_identity) \
VALUES(?1, ?2, ?3, ?4, ?5) \
ON CONFLICT(file_path) DO UPDATE SET \
session_id=excluded.session_id, \
byte_offset=excluded.byte_offset, \
updated_at=excluded.updated_at, \
file_identity=excluded.file_identity"
.into(),
params: vec![
SqlValue::Text(path_str),
session_id
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Integer(new_offset as i64),
SqlValue::Integer(now_us),
file_identity
.map(|identity| SqlValue::Text(identity.to_string()))
.unwrap_or(SqlValue::Null),
],
label: Some("session_mirror_cursor_upsert".into()),
})
.await
.map_err(|e| {
khive_storage::StorageError::driver(
khive_storage::StorageCapability::Sql,
"mirror: cursor upsert",
e,
)
})?;
Ok(())
}
async fn write_cursor_only(
runtime: &KhiveRuntime,
path: &Path,
session_id: &Option<String>,
new_offset: u64,
file_identity: Option<&str>,
) -> Result<(), RuntimeError> {
let now_us = Utc::now().timestamp_micros();
let path_str = path.to_string_lossy().into_owned();
let sql = runtime.sql();
let mut w = sql
.writer()
.await
.map_err(|e| RuntimeError::Internal(format!("mirror_file: cursor writer: {e}")))?;
w.execute(SqlStatement {
sql: "INSERT INTO session_mirror_cursor(file_path, session_id, byte_offset, updated_at, file_identity) \
VALUES(?1, ?2, ?3, ?4, ?5) \
ON CONFLICT(file_path) DO UPDATE SET \
session_id=COALESCE(excluded.session_id, session_mirror_cursor.session_id), \
byte_offset=excluded.byte_offset, \
updated_at=excluded.updated_at, \
file_identity=excluded.file_identity"
.into(),
params: vec![
SqlValue::Text(path_str),
session_id
.as_deref()
.map(|s| SqlValue::Text(s.to_string()))
.unwrap_or(SqlValue::Null),
SqlValue::Integer(new_offset as i64),
SqlValue::Integer(now_us),
file_identity
.map(|identity| SqlValue::Text(identity.to_string()))
.unwrap_or(SqlValue::Null),
],
label: Some("session_mirror_cursor_only".into()),
})
.await
.map_err(|e| RuntimeError::Internal(format!("mirror_file: cursor write: {e}")))?;
Ok(())
}
#[cfg(test)]
#[path = "ingest_tests.rs"]
mod tests;