use std::collections::{BTreeMap, BTreeSet};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::time::Duration;
use rightkit_fs::digest::Sha256Hasher;
use serde::{Deserialize, Serialize};
use crate::{field, FieldValue, ItemHeader, Ledger, RawRecord, RecordMapper};
pub const PERMANENT_SCHEMA_FAILURE_BACKOFF: Duration = Duration::from_secs(60 * 60);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StateFileNames {
pub ledger: String,
pub versions: String,
pub outbox: String,
pub legacy_tombstones: String,
pub legacy_pending: String,
pub legacy_aliases: String,
pub legacy_acks: String,
}
impl Default for StateFileNames {
fn default() -> Self {
Self {
ledger: "cloud-sync-ledger.json".to_string(),
versions: "cloud-sync-local-versions.json".to_string(),
outbox: "cloud-sync-deletions-outbox.json".to_string(),
legacy_tombstones: "cloud-sync-tombstones.json".to_string(),
legacy_pending: "cloud-sync-tombstones.pending.json".to_string(),
legacy_aliases: "cloud-sync-aliases.json".to_string(),
legacy_acks: "cloud-sync-acks.json".to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TranscriptSyncProfile {
pub record_type: String,
pub id_domain: String,
pub scope_domain: String,
pub default_source: String,
pub helper_bundle_production: String,
pub helper_bundle_development: String,
pub files: StateFileNames,
}
impl Default for TranscriptSyncProfile {
fn default() -> Self {
Self {
record_type: "Transcript".to_string(),
id_domain: "rightkit-cloud-id".to_string(),
scope_domain: "rightkit-cloud-scope".to_string(),
default_source: "Unknown".to_string(),
helper_bundle_production: "CloudKitHelper".to_string(),
helper_bundle_development: "CloudKitHelperDev".to_string(),
files: StateFileNames::default(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CloudTranscript {
pub id: String,
pub text: String,
pub raw_text: String,
pub created_at_ms: u64,
pub updated_at_ms: u64,
pub source: String,
pub tags: Vec<String>,
pub folder: Option<String>,
}
fn to_i64(v: u64) -> i64 {
v.min(i64::MAX as u64) as i64
}
fn to_u64(v: i64) -> u64 {
v.max(0) as u64
}
fn replace_file_atomically(path: &Path, bytes: &[u8]) -> io::Result<()> {
let options = rightkit_fs::atomic::AtomicWriteOptions {
create_parents: true,
..Default::default()
};
rightkit_fs::atomic::atomic_write_with_options(path, options, |file| file.write_all(bytes))
.map_err(|error| match error {
rightkit_fs::FsError::Io { source, .. }
| rightkit_fs::FsError::ReplacedButDirSyncFailed { source, .. } => source,
other => io::Error::other(other.to_string()),
})
}
const MAX_STATE_FILE_BYTES: u64 = 64 * 1024 * 1024;
fn read_state_file(path: &Path, max_bytes: u64) -> io::Result<Vec<u8>> {
let too_large = || io::Error::new(io::ErrorKind::InvalidData, "state file larger than its cap");
let file = rightkit_fs::open_read_nofollow(path, &rightkit_fs::ReadOptions::new()).map_err(
|error| match error {
rightkit_fs::FsError::Io { source, .. } => source,
_ => io::Error::new(
io::ErrorKind::InvalidData,
"state file is a link or not a regular file",
),
},
)?;
if file.metadata()?.len() > max_bytes {
return Err(too_large());
}
let mut bytes = Vec::new();
file.take(max_bytes.saturating_add(1))
.read_to_end(&mut bytes)?;
if bytes.len() as u64 > max_bytes {
return Err(too_large());
}
Ok(bytes)
}
pub struct TranscriptMapper {
profile: TranscriptSyncProfile,
}
impl TranscriptMapper {
pub fn new(profile: TranscriptSyncProfile) -> Self {
Self { profile }
}
}
impl RecordMapper for TranscriptMapper {
type Item = CloudTranscript;
fn record_type(&self) -> &str {
&self.profile.record_type
}
fn header(&self, item: &CloudTranscript) -> ItemHeader {
ItemHeader {
id: item.id.clone(),
created_at_ms: to_i64(item.created_at_ms),
updated_at_ms: to_i64(item.updated_at_ms),
origin_device: None,
}
}
fn encode(&self, item: &CloudTranscript, record: &mut RawRecord) {
record.set("text", FieldValue::Text(item.text.clone()));
record.set("rawText", FieldValue::Text(item.raw_text.clone()));
record.set("source", FieldValue::Text(item.source.clone()));
record.set_opt(
"tags",
(!item.tags.is_empty()).then(|| FieldValue::Strings(item.tags.clone())),
);
record.set_opt("folder", item.folder.clone().map(FieldValue::Text));
}
fn decode(&self, record: &RawRecord) -> Result<CloudTranscript, String> {
let raw_id = record
.text(field::ID)
.unwrap_or_else(|| record.record_name.clone());
let text = record
.get("text")
.and_then(FieldValue::as_text_or_bytes)
.ok_or_else(|| "missing text".to_string())?;
let raw_text = record
.get("rawText")
.and_then(FieldValue::as_text_or_bytes)
.ok_or_else(|| "missing rawText".to_string())?;
let fallback = record.modified_ms.or(record.created_ms);
let created = record
.date_ms(field::CREATED_AT)
.or(fallback)
.ok_or_else(|| "missing createdAt".to_string())?;
let updated = record
.date_ms(field::UPDATED_AT)
.or(fallback)
.ok_or_else(|| "missing updatedAt".to_string())?;
Ok(CloudTranscript {
id: self.profile.logical_cloud_id(&raw_id),
text,
raw_text,
created_at_ms: to_u64(created),
updated_at_ms: to_u64(updated),
source: record
.text("source")
.unwrap_or_else(|| self.profile.default_source.clone()),
tags: record
.get("tags")
.and_then(FieldValue::as_strings)
.map(<[String]>::to_vec)
.unwrap_or_default(),
folder: record.text("folder"),
})
}
}
impl TranscriptSyncProfile {
pub fn logical_cloud_id(&self, raw: &str) -> String {
if let Some(uuid) = raw.strip_prefix("cloudkit-") {
if let Some(id) = canonical_uuid(uuid) {
return id;
}
}
if let Some(uuid) = canonical_uuid(raw) {
return uuid;
}
let mut digest = Sha256Hasher::new();
digest.update(self.id_domain.as_bytes());
digest.update(b"\0");
digest.update(raw.as_bytes());
let mut bytes = [0u8; 16];
bytes.copy_from_slice(&digest.finalize_bytes()[..16]);
bytes[6] = (bytes[6] & 0x0f) | 0x50;
bytes[8] = (bytes[8] & 0x3f) | 0x80;
format!("{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
bytes[0],bytes[1],bytes[2],bytes[3],bytes[4],bytes[5],bytes[6],bytes[7],
bytes[8],bytes[9],bytes[10],bytes[11],bytes[12],bytes[13],bytes[14],bytes[15])
}
pub fn legacy_scoped_file_name(
&self,
authorized: &str,
legacy: Option<&str>,
base: &str,
) -> String {
if legacy.is_none() || legacy == Some(authorized) {
return base.to_string();
}
let mut digest = Sha256Hasher::new();
digest.update(self.scope_domain.as_bytes());
digest.update(b"\0");
digest.update(authorized.as_bytes());
let hash = digest.finalize();
format!("cloud-sync-{hash}-{base}")
}
pub fn helper_bundle_name(&self, env: CloudKitEnvironment) -> &str {
match env {
CloudKitEnvironment::Production => &self.helper_bundle_production,
CloudKitEnvironment::Development => &self.helper_bundle_development,
}
}
}
fn canonical_uuid(raw: &str) -> Option<String> {
let shaped = raw.len() == 36
&& raw.as_bytes().iter().enumerate().all(|(i, b)| {
if matches!(i, 8 | 13 | 18 | 23) {
*b == b'-'
} else {
b.is_ascii_hexdigit()
}
});
shaped.then(|| raw.to_ascii_lowercase())
}
pub fn content_hash(text: &str, raw_text: &str) -> String {
let mut h = Sha256Hasher::new();
for part in [text, raw_text] {
h.update(&(part.len() as u64).to_le_bytes());
h.update(part.as_bytes());
}
h.finalize()
}
pub fn cloud_source_is_mac(source: &str) -> bool {
matches!(
source.trim().to_ascii_lowercase().as_str(),
"mac" | "macos" | "desktop"
)
}
#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
struct VersionEntry {
#[serde(default)]
hash: Option<String>,
updated_at_ms: u64,
}
#[derive(Debug, Default)]
pub struct LocalVersions {
path: PathBuf,
entries: BTreeMap<String, VersionEntry>,
dirty: bool,
}
impl LocalVersions {
pub fn load(path: impl Into<PathBuf>) -> Self {
let path = path.into();
let entries = std::fs::read(&path)
.ok()
.and_then(|b| serde_json::from_slice(&b).ok())
.unwrap_or_default();
Self {
path,
entries,
dirty: false,
}
}
pub fn load_capped(
path: impl Into<PathBuf>,
max_bytes: u64,
max_entries: usize,
) -> Result<Self, String> {
let path = path.into();
let entries = match read_state_file(&path, max_bytes) {
Ok(bytes) => {
serde_json::from_slice::<BTreeMap<String, VersionEntry>>(&bytes).unwrap_or_default()
}
Err(error) if error.kind() == io::ErrorKind::NotFound => BTreeMap::new(),
Err(error) => return Err(format!("read cloud sync versions: {error}")),
};
if entries.len() > max_entries {
return Err("cloud sync versions hold more entries than the cap".to_string());
}
Ok(Self {
path,
entries,
dirty: false,
})
}
pub fn observe(&mut self, id: &str, hash: &str, created_ms: u64, now_ms: u64) -> u64 {
match self.entries.get_mut(id) {
None => {
self.entries.insert(
id.to_string(),
VersionEntry {
hash: Some(hash.to_string()),
updated_at_ms: created_ms,
},
);
self.dirty = true;
created_ms
}
Some(e) if e.hash.is_none() => {
e.hash = Some(hash.to_string());
self.dirty = true;
e.updated_at_ms
}
Some(e) if e.hash.as_deref() == Some(hash) => e.updated_at_ms,
Some(e) => {
e.updated_at_ms = now_ms.max(e.updated_at_ms.saturating_add(1));
e.hash = Some(hash.to_string());
self.dirty = true;
e.updated_at_ms
}
}
}
pub fn applied(&mut self, id: &str, hash: &str, updated_at_ms: u64) {
self.entries.insert(
id.to_string(),
VersionEntry {
hash: Some(hash.to_string()),
updated_at_ms,
},
);
self.dirty = true;
}
pub fn seed(&mut self, id: &str, updated_at_ms: u64) {
self.entries.entry(id.to_string()).or_insert(VersionEntry {
hash: None,
updated_at_ms,
});
self.dirty = true;
}
pub fn retain_ids(&mut self, live: &BTreeSet<String>) {
let before = self.entries.len();
self.entries.retain(|id, _| live.contains(id));
if self.entries.len() != before {
self.dirty = true;
}
}
pub fn save_if_dirty(&mut self) -> Result<(), String> {
if !self.dirty {
return Ok(());
}
let bytes = serde_json::to_vec(&self.entries).map_err(|e| e.to_string())?;
replace_file_atomically(&self.path, &bytes)
.map_err(|e| format!("write cloud sync versions: {e}"))?;
self.dirty = false;
Ok(())
}
}
pub fn read_outbox(path: &Path) -> Result<BTreeMap<String, u64>, String> {
match read_state_file(path, MAX_STATE_FILE_BYTES) {
Ok(b) => {
if outbox_is_versioned(&b) {
return Err("deletion outbox is in the scoped format; not readable as v1".into());
}
serde_json::from_slice(&b).map_err(|e| format!("parse deletion outbox: {e}"))
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(BTreeMap::new()),
Err(e) => Err(format!("read deletion outbox: {e}")),
}
}
pub fn write_outbox(path: &Path, entries: &BTreeMap<String, u64>) -> Result<(), String> {
match read_state_file(path, MAX_STATE_FILE_BYTES) {
Ok(b) if outbox_is_versioned(&b) => {
return Err("deletion outbox is in the scoped format; not overwritten as v1".into());
}
Ok(_) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(format!("read deletion outbox: {e}")),
}
let bytes = serde_json::to_vec(entries).map_err(|e| e.to_string())?;
replace_file_atomically(path, &bytes).map_err(|e| format!("write deletion outbox: {e}"))
}
fn outbox_is_versioned(bytes: &[u8]) -> bool {
serde_json::from_slice::<serde_json::Value>(bytes)
.ok()
.is_some_and(|value| value.get("v").is_some())
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ConsentScopeBinding {
#[serde(default)]
pub authorized_scope: Option<String>,
#[serde(default)]
pub legacy_scope: Option<String>,
}
pub fn authorize_scope(
authorized: &mut Option<String>,
legacy: &mut Option<String>,
acknowledged: Option<&str>,
actual: &str,
explicit: bool,
account_changed_message: &str,
) -> Result<(), String> {
if actual.is_empty() {
return Err("cloud sync scope lookup returned an empty scope".to_string());
}
if legacy.is_none() {
*legacy = authorized
.clone()
.or_else(|| acknowledged.map(str::to_string))
.or_else(|| Some(actual.to_string()));
}
if explicit {
*authorized = Some(actual.to_string());
} else if authorized.is_none() {
*authorized = Some(acknowledged.unwrap_or(actual).to_string());
}
if authorized.as_deref() != Some(actual) {
return Err(account_changed_message.to_string());
}
Ok(())
}
#[derive(Debug, Clone)]
pub struct LegacyFiles {
pub tombstones: PathBuf,
pub pending: PathBuf,
pub aliases: PathBuf,
pub acks: PathBuf,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MigrationReport {
pub tombstones: usize,
pub pending_tombstones: usize,
pub alias_ids: usize,
pub seeded_versions: usize,
pub dropped_conflicts: usize,
}
#[derive(Deserialize)]
struct LegacyTombstone {
id: String,
#[serde(default, alias = "deletedAt")]
deleted_at_ms: u64,
#[serde(default)]
aliases: Vec<String>,
}
#[derive(Deserialize)]
struct LegacyAlias {
id: String,
#[serde(default)]
aliases: Vec<String>,
}
#[derive(Deserialize, Default)]
struct LegacyAckEntry {
#[serde(default)]
updated_at_ms: u64,
}
#[derive(Deserialize, Default)]
struct LegacyAcks {
#[serde(default)]
scope: Option<String>,
#[serde(default)]
entries: BTreeMap<String, LegacyAckEntry>,
#[serde(default)]
conflicts: BTreeMap<String, serde_json::Value>,
}
fn read_legacy<T: serde::de::DeserializeOwned>(
path: &Path,
what: &str,
) -> Result<Option<T>, String> {
match read_state_file(path, MAX_STATE_FILE_BYTES) {
Ok(b) => serde_json::from_slice(&b)
.map(Some)
.map_err(|e| format!("parse legacy {what}: {e}")),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(format!("read legacy {what}: {e}")),
}
}
pub type MigratedLegacy = (Ledger, BTreeMap<String, u64>, MigrationReport);
impl TranscriptSyncProfile {
pub fn migrate_legacy(
&self,
files: &LegacyFiles,
scope: Option<&str>,
) -> Result<Option<MigratedLegacy>, String> {
let scope = scope.filter(|value| !value.is_empty());
let acks = read_legacy::<LegacyAcks>(&files.acks, "acks")
.ok()
.flatten()
.filter(|acks| scope.is_some() && acks.scope.as_deref() == scope);
let proven = |path: &Path, base: &str| {
scope.is_some_and(|scope| {
let scoped = self.legacy_scoped_file_name(scope, Some(""), base);
path.file_name().is_some_and(|name| {
name == std::ffi::OsStr::new(&scoped)
|| (acks.is_some() && name == std::ffi::OsStr::new(base))
})
})
};
let tombs: Option<Vec<LegacyTombstone>> =
if proven(&files.tombstones, &self.files.legacy_tombstones) {
read_legacy(&files.tombstones, "tombstones")?
} else {
None
};
let pending: Option<Vec<LegacyTombstone>> =
if proven(&files.pending, &self.files.legacy_pending) {
read_legacy(&files.pending, "pending tombstones")?
} else {
None
};
let aliases: Option<Vec<LegacyAlias>> =
if proven(&files.aliases, &self.files.legacy_aliases) {
read_legacy(&files.aliases, "aliases")?
} else {
None
};
if tombs.is_none() && pending.is_none() && aliases.is_none() && acks.is_none() {
return Ok(None);
}
let mut ledger = Ledger::default();
let mut report = MigrationReport::default();
ledger.scope = scope.map(str::to_string);
let add_aliases = |ledger: &mut Ledger, id: &str, names: &[String]| {
let set = ledger.aliases.entry(id.to_string()).or_default();
for n in names {
set.insert(n.clone());
}
};
for t in tombs.iter().flatten() {
let id = self.logical_cloud_id(&t.id);
let at = to_i64(t.deleted_at_ms);
let e = ledger.tombstones.entry(id.clone()).or_insert(at);
*e = (*e).max(at);
add_aliases(&mut ledger, &id, &t.aliases);
}
for t in pending.iter().flatten() {
let id = self.logical_cloud_id(&t.id);
let at = to_i64(t.deleted_at_ms);
let e = ledger.tombstones.entry(id.clone()).or_insert(at);
*e = (*e).max(at);
ledger.pending_tombstones.insert(id.clone());
add_aliases(&mut ledger, &id, &t.aliases);
}
for a in aliases.iter().flatten() {
add_aliases(&mut ledger, &self.logical_cloud_id(&a.id), &a.aliases);
}
let mut seeds = BTreeMap::new();
if let Some(a) = &acks {
report.dropped_conflicts = a.conflicts.len();
for (id, entry) in &a.entries {
if entry.updated_at_ms > 0 && !ledger.tombstones.contains_key(id) {
seeds.insert(id.clone(), entry.updated_at_ms);
}
}
}
report.tombstones = ledger.tombstones.len();
report.pending_tombstones = ledger.pending_tombstones.len();
report.alias_ids = ledger.aliases.len();
report.seeded_versions = seeds.len();
Ok(Some((ledger, seeds, report)))
}
pub fn migrate_legacy_once(
&self,
files: &LegacyFiles,
scope: Option<&str>,
ledger_path: &Path,
versions_path: &Path,
) -> Result<Option<MigrationReport>, String> {
let _ledger_lock = crate::storage::ledger_file_lock(ledger_path)?;
if ledger_path.exists() {
return Ok(None);
}
let Some((ledger, seeds, mut report)) = self.migrate_legacy(files, scope)? else {
return Ok(None);
};
report.seeded_versions = 0;
if !seeds.is_empty() {
let entries = match read_state_file(versions_path, MAX_STATE_FILE_BYTES) {
Ok(bytes) => serde_json::from_slice::<BTreeMap<String, VersionEntry>>(&bytes)
.map_err(|error| format!("parse existing cloud sync versions: {error}"))?,
Err(error) if error.kind() == io::ErrorKind::NotFound => BTreeMap::new(),
Err(error) => return Err(format!("read existing cloud sync versions: {error}")),
};
let mut versions = LocalVersions {
path: versions_path.to_path_buf(),
entries,
dirty: false,
};
for (id, at) in &seeds {
if !versions.entries.contains_key(id) {
versions.seed(id, *at);
report.seeded_versions += 1;
}
}
versions.save_if_dirty()?;
}
let bytes = serde_json::to_vec(&ledger).map_err(|e| e.to_string())?;
replace_file_atomically(ledger_path, &bytes)
.map_err(|e| format!("write migrated cloud sync ledger: {e}"))?;
Ok(Some(report))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CloudKitEnvironment {
Production,
Development,
}
const ENTITLEMENT_KEY: &str = "com.apple.developer.icloud-container-environment";
pub fn requested_environment() -> Result<CloudKitEnvironment, String> {
Ok(CloudKitEnvironment::Production)
}
pub fn parse_entitlement_environment(xml: &str) -> Option<CloudKitEnvironment> {
let after = xml.split(ENTITLEMENT_KEY).nth(1)?;
let value = after.split("<string>").nth(1)?.split("</string>").next()?;
match value.trim() {
"Development" => Some(CloudKitEnvironment::Development),
"Production" => Some(CloudKitEnvironment::Production),
_ => None,
}
}
pub fn check_helper_environment(
expected: CloudKitEnvironment,
signed: Option<CloudKitEnvironment>,
) -> Result<(), String> {
match (expected, signed) {
(CloudKitEnvironment::Development, Some(CloudKitEnvironment::Development)) => Ok(()),
(CloudKitEnvironment::Development, other) => Err(format!(
"a Development CloudKit helper is required; this one is signed {other:?}"
)),
(CloudKitEnvironment::Production, Some(CloudKitEnvironment::Development)) => {
Err("release build refuses a CloudKit helper signed for Development".to_string())
}
(CloudKitEnvironment::Production, _) => Ok(()),
}
}
pub fn env_file_name(base: &str, env: CloudKitEnvironment) -> String {
match env {
CloudKitEnvironment::Production => base.to_string(),
CloudKitEnvironment::Development => match base.rsplit_once('.') {
Some((stem, ext)) => format!("{stem}-dev.{ext}"),
None => format!("{base}-dev"),
},
}
}
pub fn native_observer_allowed(env: CloudKitEnvironment) -> bool {
env == CloudKitEnvironment::Production
}
pub fn is_production_schema_error(error: &str) -> bool {
let error = error.to_ascii_lowercase();
error.contains("cannot create new type") && error.contains("production schema")
}
pub fn error_is_terminal(error: &str) -> bool {
let lower = error.to_ascii_lowercase();
is_production_schema_error(error)
|| [
"not authenticated",
"not_authenticated",
"missing entitlement",
"missing_entitlement",
"permission failure",
"permission_failure",
"account unavailable",
"account changed",
]
.iter()
.any(|marker| lower.contains(marker))
}