pub mod filesystem;
pub mod postgres;
pub mod sql_dialect;
pub mod sqlite;
use anyhow::Result;
use async_trait::async_trait;
use std::collections::{HashMap, HashSet};
use uuid::Uuid;
#[derive(Debug, Clone)]
pub struct SourceDocument {
pub id: Uuid,
pub content: String,
pub content_len: i32,
pub filename: Option<String>,
}
#[derive(Debug, Clone)]
pub struct SourceDupeMatch {
pub child_id: Uuid,
pub parent_id: Uuid,
pub jaccard_similarity: f64,
pub size_difference: i32,
pub size_difference_pct: f64,
}
#[async_trait]
pub trait DocumentSource: Send + Sync {
async fn source_name(&self) -> Result<String> {
Ok("unknown".to_string())
}
async fn count_total(&self) -> Result<i64> {
anyhow::bail!("count_total is not implemented for this DocumentSource")
}
async fn count_unprocessed(&self) -> Result<i64> {
self.count_total().await
}
async fn fetch_all_after(
&self,
last_id: Option<Uuid>,
limit: i64,
) -> Result<Vec<SourceDocument>> {
let _ = (last_id, limit);
anyhow::bail!("fetch_all_after is not implemented for this DocumentSource")
}
async fn fetch_unprocessed_ids_after(
&self,
last_id: Option<Uuid>,
limit: i64,
) -> Result<Vec<Uuid>> {
let docs = self.fetch_all_after(last_id, limit).await?;
Ok(docs.into_iter().map(|d| d.id).collect())
}
async fn fetch_by_ids(&self, ids: &[Uuid]) -> Result<Vec<SourceDocument>> {
let _ = ids;
anyhow::bail!("fetch_by_ids is not implemented for this DocumentSource")
}
async fn fetch_existing_parent_ids(&self, _ids: &[Uuid]) -> Result<HashSet<Uuid>> {
Ok(HashSet::new())
}
async fn fetch_existing_dupe_parents(
&self,
_child_ids: &[Uuid],
) -> Result<HashMap<Uuid, Uuid>> {
Ok(HashMap::new())
}
async fn mark_as_parents(&self, _ids: &[Uuid]) -> Result<u64> {
Ok(0)
}
async fn mark_as_children(&self, _ids: &[Uuid]) -> Result<u64> {
Ok(0)
}
async fn write_dupes(&self, _matches: &[SourceDupeMatch]) -> Result<u64> {
Ok(0)
}
fn supports_write(&self) -> bool {
false
}
fn tracks_state(&self) -> bool {
false
}
}
pub use filesystem::FileSystemSource;
pub use postgres::PostgresSource;
pub use sql_dialect::SqlDialect;
pub use sqlite::SqliteSource;