1use crate::brain::ImageThreadCredentialSnapshot;
5use crate::storage::{
6 self, MessageAttachment, SideChatData, SideChatMetadata, ThreadData, ThreadMessage,
7 ThreadMetadata, ThreadStorage, DEFAULT_SIDE_CHAT_TITLE, DEFAULT_THREAD_TITLE,
8};
9use chrono::Utc;
10use serde::Serialize;
11use std::collections::BTreeMap;
12use std::path::Path;
13use std::sync::atomic::{AtomicU64, Ordering};
14use std::sync::{Arc, Mutex, OnceLock};
15use std::time::{Duration, Instant};
16
17use crate::services::brain;
18use crate::settings;
19
20pub type ThreadResult<T> = std::result::Result<T, String>;
21
22#[derive(Serialize)]
23#[serde(rename_all = "camelCase")]
24pub struct ImageThreadCreation {
25 pub thread_id: String,
26 pub brain_job_id: Option<String>,
27 pub ocr_job_id: Option<String>,
28}
29
30#[derive(Serialize)]
31#[serde(rename_all = "camelCase")]
32pub struct SideChatCreation {
33 pub sidechat_id: String,
34 pub title: String,
35}
36
37#[derive(Clone)]
38struct BrainJobRecord {
39 job_id: String,
40 thread_id: String,
41 status: String,
42 phase: String,
43 error: Option<String>,
44}
45
46struct PendingImageThreadCredential {
47 credential: Option<ImageThreadCredentialSnapshot>,
48 captured_at: Instant,
49}
50
51#[derive(Serialize)]
52#[serde(rename_all = "camelCase")]
53pub struct BrainJobSnapshot {
54 pub job_id: String,
55 pub thread_id: String,
56 pub status: String,
57 pub phase: String,
58 pub error: Option<String>,
59}
60
61impl From<&BrainJobRecord> for BrainJobSnapshot {
62 fn from(job: &BrainJobRecord) -> Self {
63 Self {
64 job_id: job.job_id.clone(),
65 thread_id: job.thread_id.clone(),
66 status: job.status.clone(),
67 phase: job.phase.clone(),
68 error: job.error.clone(),
69 }
70 }
71}
72
73static BRAIN_JOBS: OnceLock<Arc<Mutex<BTreeMap<u64, BrainJobRecord>>>> = OnceLock::new();
74static NEXT_BRAIN_JOB_ID: AtomicU64 = AtomicU64::new(1);
75static PENDING_IMAGE_THREAD_CREDENTIALS: OnceLock<
76 Mutex<BTreeMap<String, PendingImageThreadCredential>>,
77> = OnceLock::new();
78static NEXT_IMAGE_THREAD_CREATION_ID: AtomicU64 = AtomicU64::new(1);
79static THREAD_INDEX_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
80const IMAGE_THREAD_CREATION_TTL: Duration = Duration::from_secs(10 * 60);
81
82pub(super) fn active_storage() -> ThreadResult<ThreadStorage> {
83 storage::thread_store().map_err(|error| error.to_string())
84}
85
86fn brain_jobs() -> &'static Arc<Mutex<BTreeMap<u64, BrainJobRecord>>> {
87 BRAIN_JOBS.get_or_init(|| Arc::new(Mutex::new(BTreeMap::new())))
88}
89
90fn pending_image_thread_credentials(
91) -> &'static Mutex<BTreeMap<String, PendingImageThreadCredential>> {
92 PENDING_IMAGE_THREAD_CREDENTIALS.get_or_init(|| Mutex::new(BTreeMap::new()))
93}
94
95fn lock_brain_jobs(
96 jobs: &Mutex<BTreeMap<u64, BrainJobRecord>>,
97) -> ThreadResult<std::sync::MutexGuard<'_, BTreeMap<u64, BrainJobRecord>>> {
98 jobs.lock()
99 .map_err(|_| "Thread job state is unavailable".to_string())
100}
101
102fn thread_index_lock() -> &'static Mutex<()> {
103 THREAD_INDEX_LOCK.get_or_init(|| Mutex::new(()))
104}
105
106fn lock_pending_image_thread_credentials(
107) -> ThreadResult<std::sync::MutexGuard<'static, BTreeMap<String, PendingImageThreadCredential>>> {
108 pending_image_thread_credentials()
109 .lock()
110 .map_err(|_| "Image thread credential snapshots are unavailable".to_string())
111}
112
113pub async fn prepare_image_thread_creation() -> ThreadResult<String> {
114 let credential = brain().capture_image_thread_credential().await?;
115 let sequence = NEXT_IMAGE_THREAD_CREATION_ID.fetch_add(1, Ordering::Relaxed);
116 let creation_id = format!("image-thread-creation-{sequence}");
117 let now = Instant::now();
118 let mut pending = lock_pending_image_thread_credentials()?;
119 pending.retain(|_, entry| {
120 now.saturating_duration_since(entry.captured_at) <= IMAGE_THREAD_CREATION_TTL
121 });
122 pending.insert(
123 creation_id.clone(),
124 PendingImageThreadCredential {
125 credential,
126 captured_at: now,
127 },
128 );
129 drop(pending);
130 let cleanup_creation_id = creation_id.clone();
131 tokio::spawn(async move {
132 tokio::time::sleep(IMAGE_THREAD_CREATION_TTL).await;
133 let _ = cancel_image_thread_creation(&cleanup_creation_id);
134 });
135 Ok(creation_id)
136}
137
138pub fn cancel_image_thread_creation(creation_id: &str) -> ThreadResult<()> {
139 if let Some(pending) = PENDING_IMAGE_THREAD_CREDENTIALS.get() {
140 pending
141 .lock()
142 .map_err(|_| "Image thread credential snapshots are unavailable".to_string())?
143 .remove(creation_id);
144 }
145 Ok(())
146}
147
148fn take_image_thread_credential(
149 creation_id: &str,
150) -> ThreadResult<Option<ImageThreadCredentialSnapshot>> {
151 let entry = lock_pending_image_thread_credentials()?
152 .remove(creation_id)
153 .ok_or_else(|| {
154 "The image thread creation snapshot expired or was already used".to_string()
155 })?;
156 if entry.captured_at.elapsed() > IMAGE_THREAD_CREATION_TTL {
157 return Err("The image thread creation snapshot expired".to_string());
158 }
159 Ok(entry.credential)
160}
161
162fn is_supported_image(path: &Path) -> bool {
163 path.extension()
164 .and_then(|extension| extension.to_str())
165 .map(str::to_ascii_lowercase)
166 .is_some_and(|extension| {
167 matches!(
168 extension.as_str(),
169 "png" | "jpg" | "jpeg" | "gif" | "webp" | "bmp" | "svg"
170 )
171 })
172}
173
174fn create_thread_on_disk(
175 source_path: &str,
176 workspace_id: Option<&str>,
177) -> ThreadResult<(String, String)> {
178 let source = Path::new(source_path);
179 if !source.is_file() {
180 return Err("The pasted image file could not be found".to_string());
181 }
182 if !is_supported_image(source) {
183 return Err("Thread creation requires a supported image file".to_string());
184 }
185 let image_tone = lens::detect_image_tone_from_path(source)?;
186
187 let _index_guard = thread_index_lock()
188 .lock()
189 .map_err(|_| "Thread index is unavailable".to_string())?;
190 let storage = active_storage()?;
191 let stored = storage
192 .store_image_from_path(source_path, image_tone)
193 .map_err(|error| error.to_string())?;
194 let display_name = source
195 .file_name()
196 .and_then(|name| name.to_str())
197 .unwrap_or("pasted-image.png");
198 let metadata = ThreadMetadata::new(DEFAULT_THREAD_TITLE.to_string(), stored.hash);
199 let initial_attachment = storage
200 .attachment_manifest_entry(&metadata.image_hash, display_name, Utc::now())
201 .map_err(|error| error.to_string())?;
202 let thread = ThreadData::new(metadata.clone(), initial_attachment);
203 match workspace_id.map(str::trim).filter(|id| !id.is_empty()) {
204 Some(workspace_id) => storage
205 .save_thread_in_workspace(&thread, workspace_id)
206 .map_err(|error| error.to_string())?,
207 None => storage
208 .save_thread(&thread)
209 .map_err(|error| error.to_string())?,
210 }
211 Ok((metadata.id, stored.path))
212}
213
214fn update_brain_job(
215 jobs: &Mutex<BTreeMap<u64, BrainJobRecord>>,
216 sequence: u64,
217 status: &str,
218 phase: &str,
219 error: Option<String>,
220) {
221 if let Ok(mut records) = lock_brain_jobs(jobs) {
222 if let Some(job) = records.get_mut(&sequence) {
223 job.status = status.to_string();
224 job.phase = phase.to_string();
225 job.error = error;
226 }
227 }
228}
229
230fn persist_generated_title(thread_id: &str, title: &str) -> ThreadResult<()> {
231 let title = title.trim();
232 if title.is_empty() {
233 return Err("The generated thread title was empty".to_string());
234 }
235 let _index_guard = thread_index_lock()
236 .lock()
237 .map_err(|_| "Thread index is unavailable".to_string())?;
238 let storage = active_storage()?;
239 let mut thread = storage
240 .load_thread(thread_id)
241 .map_err(|error| error.to_string())?;
242 if thread.metadata.title != DEFAULT_THREAD_TITLE {
243 return Ok(());
244 }
245 thread.metadata.title = title.to_string();
246 storage
247 .update_thread_metadata(&thread.metadata)
248 .map_err(|error| error.to_string())
249}
250
251async fn run_brain_job(
252 sequence: u64,
253 jobs: Arc<Mutex<BTreeMap<u64, BrainJobRecord>>>,
254 thread_id: String,
255 image_path: String,
256 model: String,
257 effort: String,
258 credential: ImageThreadCredentialSnapshot,
259) {
260 update_brain_job(&jobs, sequence, "running", "uploading", None);
261 let result = async {
262 let uploaded = brain()
263 .ensure_thread_image_uploaded_with_snapshot(&credential, image_path)
264 .await?;
265 update_brain_job(&jobs, sequence, "running", "generating-title", None);
266 let model_candidates = brain().build_model_attempt_plan(model, effort).await?;
267 let title = brain()
268 .suggest_thread_title_from_file_with_snapshot(&credential, uploaded, model_candidates)
269 .await?;
270 let title_thread_id = thread_id.clone();
271 tokio::task::spawn_blocking(move || persist_generated_title(&title_thread_id, &title))
272 .await
273 .map_err(|error| format!("Thread title save task failed: {error}"))??;
274 Ok::<(), String>(())
275 }
276 .await;
277
278 match result {
279 Ok(()) => update_brain_job(&jobs, sequence, "completed", "completed", None),
280 Err(error) => update_brain_job(&jobs, sequence, "failed", "failed", Some(error)),
281 }
282}
283
284fn start_brain_job(
285 thread_id: String,
286 image_path: String,
287 model: String,
288 effort: String,
289 credential: ImageThreadCredentialSnapshot,
290) -> ThreadResult<String> {
291 let sequence = NEXT_BRAIN_JOB_ID.fetch_add(1, Ordering::Relaxed);
292 let job_id = format!("brain-{sequence}");
293 let jobs = brain_jobs();
294 lock_brain_jobs(jobs)?.insert(
295 sequence,
296 BrainJobRecord {
297 job_id: job_id.clone(),
298 thread_id: thread_id.clone(),
299 status: "queued".to_string(),
300 phase: "stored".to_string(),
301 error: None,
302 },
303 );
304 let worker_jobs = Arc::clone(jobs);
305 let crash_jobs = Arc::clone(jobs);
306 let worker = tokio::spawn(async move {
307 run_brain_job(
308 sequence,
309 worker_jobs,
310 thread_id,
311 image_path,
312 model,
313 effort,
314 credential,
315 )
316 .await;
317 });
318 tokio::spawn(async move {
319 if let Err(error) = worker.await {
320 update_brain_job(
321 &crash_jobs,
322 sequence,
323 "failed",
324 "failed",
325 Some(format!("Thread job worker crashed: {error}")),
326 );
327 }
328 });
329 Ok(job_id)
330}
331
332pub async fn create_image_thread(
333 creation_id: String,
334 source_path: String,
335 workspace_id: Option<String>,
336) -> ThreadResult<ImageThreadCreation> {
337 let credential = take_image_thread_credential(&creation_id)?;
338 let config = settings::load_config()?;
339 let creation_workspace_id = workspace_id.clone();
340 let (thread_id, image_path) = tokio::task::spawn_blocking(move || {
341 create_thread_on_disk(&source_path, creation_workspace_id.as_deref())
342 })
343 .await
344 .map_err(|error| format!("Thread creation task failed: {error}"))??;
345
346 let brain_job_id = credential
347 .map(|credential| {
348 start_brain_job(
349 thread_id.clone(),
350 image_path,
351 config.model,
352 config.effort,
353 credential,
354 )
355 })
356 .transpose()?;
357 let ocr_job_id = config
358 .ocr_enabled
359 .then(|| ocr::start_ocr_thread_job(&thread_id, &config.ocr_language))
360 .transpose()?;
361 Ok(ImageThreadCreation {
362 thread_id,
363 brain_job_id,
364 ocr_job_id,
365 })
366}
367
368pub async fn create_sidechat_thread(
369 message_markdown: String,
370 attachment_hashes: Vec<String>,
371 human_text: Option<String>,
372) -> ThreadResult<SideChatCreation> {
373 let human_text = human_text
374 .map(|value| value.trim().to_string())
375 .filter(|value| !value.is_empty());
376 let (sidechat_id, initial_title) = tokio::task::spawn_blocking(move || {
377 let _index_guard = thread_index_lock()
378 .lock()
379 .map_err(|_| "Thread index is unavailable".to_string())?;
380 let storage = active_storage()?;
381 let metadata = SideChatMetadata::new(DEFAULT_SIDE_CHAT_TITLE.to_string());
382 let attachments = attachment_hashes
383 .into_iter()
384 .map(|attachment_hash| MessageAttachment {
385 attachment_hash,
386 source_path: None,
387 })
388 .collect();
389 let message = ThreadMessage::user_with_attachments(message_markdown, attachments);
390 let sidechat = SideChatData::new(metadata.clone(), message);
391 storage
392 .save_sidechat(&sidechat)
393 .map_err(|error| error.to_string())?;
394 Ok::<_, String>((metadata.id, metadata.title))
395 })
396 .await
397 .map_err(|error| format!("SideChat creation task failed: {error}"))??;
398
399 let Some(title_source) = human_text else {
400 return Ok(SideChatCreation {
401 sidechat_id,
402 title: initial_title,
403 });
404 };
405
406 let generated_title = async {
407 let config = settings::load_config()?;
408 let candidates = brain()
409 .build_model_attempt_plan(config.model, config.effort)
410 .await?;
411 brain()
412 .suggest_thread_title_from_text(title_source, candidates)
413 .await
414 }
415 .await;
416
417 let title = match generated_title {
418 Ok(title) if !title.trim().is_empty() => {
419 let persisted_title = title.trim().to_string();
420 let metadata_id = sidechat_id.clone();
421 let next_title = persisted_title.clone();
422 tokio::task::spawn_blocking(move || {
423 let _index_guard = thread_index_lock()
424 .lock()
425 .map_err(|_| "Thread index is unavailable".to_string())?;
426 let storage = active_storage()?;
427 let mut sidechat = storage
428 .load_sidechat(&metadata_id)
429 .map_err(|error| error.to_string())?;
430 sidechat.metadata.title = next_title;
431 sidechat.metadata.updated_at = Utc::now();
432 storage
433 .update_sidechat_metadata(&sidechat.metadata)
434 .map_err(|error| error.to_string())
435 })
436 .await
437 .map_err(|error| format!("SideChat title save task failed: {error}"))??;
438 persisted_title
439 }
440 _ => initial_title,
441 };
442
443 Ok(SideChatCreation { sidechat_id, title })
444}
445
446pub fn append_sidechat_message(
447 sidechat_id: &str,
448 message_markdown: String,
449 attachment_hashes: Vec<String>,
450) -> ThreadResult<()> {
451 let _index_guard = thread_index_lock()
452 .lock()
453 .map_err(|_| "Thread index is unavailable".to_string())?;
454 let storage = active_storage()?;
455 let mut sidechat = storage
456 .load_sidechat(sidechat_id)
457 .map_err(|error| error.to_string())?;
458 let attachments = attachment_hashes
459 .into_iter()
460 .map(|attachment_hash| MessageAttachment {
461 attachment_hash,
462 source_path: None,
463 })
464 .collect();
465 sidechat.messages.push(ThreadMessage::user_with_attachments(
466 message_markdown,
467 attachments,
468 ));
469 sidechat.metadata.updated_at = Utc::now();
470 storage
471 .save_sidechat(&sidechat)
472 .map_err(|error| error.to_string())
473}
474
475pub fn get_thread_jobs_snapshot() -> ThreadResult<Vec<BrainJobSnapshot>> {
476 if let Some(jobs) = BRAIN_JOBS.get() {
477 Ok(lock_brain_jobs(jobs)?
478 .values()
479 .map(BrainJobSnapshot::from)
480 .collect())
481 } else {
482 Ok(Vec::new())
483 }
484}
485
486pub mod lens {
487 use crate::auth::{get_decrypted_api_key, session_api_keys_active, ApiKeyProvider};
488 use crate::storage::{self, OcrAnnotationEntry, ReverseImageSearchCache, ThreadStorage};
489 use image::{imageops, GenericImageView};
490 use serde::{Deserialize, Serialize};
491 use std::{
492 collections::HashMap,
493 path::Path,
494 sync::{
495 atomic::{AtomicU64, Ordering},
496 Mutex, OnceLock,
497 },
498 };
499 use tokio::sync::watch;
500 use url::Url;
501
502 use super::{active_storage, ThreadResult};
503
504 struct ReverseSearchCancellation {
505 id: u64,
506 sender: watch::Sender<bool>,
507 }
508
509 static REVERSE_SEARCH_CANCELLATIONS: OnceLock<
510 Mutex<HashMap<String, ReverseSearchCancellation>>,
511 > = OnceLock::new();
512 static NEXT_REVERSE_SEARCH_ID: AtomicU64 = AtomicU64::new(1);
513
514 fn reverse_search_cancellations() -> &'static Mutex<HashMap<String, ReverseSearchCancellation>>
515 {
516 REVERSE_SEARCH_CANCELLATIONS.get_or_init(|| Mutex::new(HashMap::new()))
517 }
518
519 fn begin_reverse_search(thread_id: &str) -> ThreadResult<(u64, watch::Receiver<bool>)> {
520 let id = NEXT_REVERSE_SEARCH_ID.fetch_add(1, Ordering::Relaxed);
521 let (sender, receiver) = watch::channel(false);
522 let previous = reverse_search_cancellations()
523 .lock()
524 .map_err(|_| "Reverse image search cancellation lock poisoned".to_string())?
525 .insert(
526 thread_id.to_string(),
527 ReverseSearchCancellation { id, sender },
528 );
529 if let Some(previous) = previous {
530 let _ = previous.sender.send(true);
531 }
532 Ok((id, receiver))
533 }
534
535 fn finish_reverse_search(thread_id: &str, id: u64) -> ThreadResult<()> {
536 let mut cancellations = reverse_search_cancellations()
537 .lock()
538 .map_err(|_| "Reverse image search cancellation lock poisoned".to_string())?;
539 if cancellations
540 .get(thread_id)
541 .is_some_and(|cancellation| cancellation.id == id)
542 {
543 cancellations.remove(thread_id);
544 }
545 Ok(())
546 }
547
548 pub fn cancel_reverse_image_search(thread_id: &str) -> ThreadResult<()> {
549 let cancellation = reverse_search_cancellations()
550 .lock()
551 .map_err(|_| "Reverse image search cancellation lock poisoned".to_string())?
552 .remove(thread_id);
553 if let Some(cancellation) = cancellation {
554 let _ = cancellation.sender.send(true);
555 }
556 Ok(())
557 }
558
559 #[derive(Serialize)]
560 #[serde(rename_all = "camelCase")]
561 pub struct ReverseImageSearchOutcome {
562 pub imgbb_url: String,
563 pub google_lens_url: String,
564 pub opened_url: String,
565 }
566
567 fn required_text<'a>(text: &'a str, message: &str) -> ThreadResult<&'a str> {
568 let text = text.trim();
569 if text.is_empty() {
570 Err(message.to_string())
571 } else {
572 Ok(text)
573 }
574 }
575
576 fn translate_url(text: &str) -> ThreadResult<String> {
577 let mut url =
578 Url::parse("https://translate.google.com/").map_err(|error| error.to_string())?;
579 url.query_pairs_mut()
580 .append_pair("text", required_text(text, "Translation text is required")?)
581 .append_pair("sl", "auto")
582 .append_pair("tl", "en")
583 .append_pair("op", "translate");
584 Ok(url.into())
585 }
586
587 pub fn search_text_url(text: &str) -> ThreadResult<String> {
588 let mut url =
589 Url::parse("https://www.google.com/search").map_err(|error| error.to_string())?;
590 url.query_pairs_mut()
591 .append_pair("q", required_text(text, "Search text is required")?);
592 Ok(url.into())
593 }
594
595 pub fn translate_text_url(text: &str) -> ThreadResult<String> {
596 translate_url(text)
597 }
598
599 fn latest_ocr_text(thread_id: &str, storage: &ThreadStorage) -> ThreadResult<String> {
600 let thread = storage
601 .load_thread(thread_id)
602 .map_err(|error| error.to_string())?;
603 let latest = thread
604 .ocr_data
605 .values()
606 .filter_map(|entry| match entry {
607 OcrAnnotationEntry::Model(model) => model
608 .scanned_at
609 .as_ref()
610 .map(|scanned_at| (scanned_at, &model.ocr_data)),
611 OcrAnnotationEntry::EmptyState(_) => None,
612 })
613 .max_by(|left, right| left.0.cmp(right.0))
614 .ok_or_else(|| "OCR has not completed for this thread".to_string())?;
615 let text = latest
616 .1
617 .iter()
618 .map(|region| region.text.trim())
619 .filter(|text| !text.is_empty())
620 .collect::<Vec<_>>()
621 .join(" ");
622 required_text(&text, "OCR completed without translatable text").map(str::to_string)
623 }
624
625 pub fn translate_thread_image_url(thread_id: &str) -> ThreadResult<String> {
626 let storage = active_storage()?;
627 translate_url(&latest_ocr_text(thread_id, &storage)?)
628 }
629
630 fn lens_url(image_url: &str) -> ThreadResult<String> {
631 let mut url =
632 Url::parse("https://lens.google.com/uploadbyurl").map_err(|error| error.to_string())?;
633 url.query_pairs_mut()
634 .append_pair("url", required_text(image_url, "ImgBB URL is required")?)
635 .append_pair("ep", "subb")
636 .append_pair("re", "df")
637 .append_pair("s", "4")
638 .append_pair("hl", "en")
639 .append_pair("gl", "US");
640 Ok(url.into())
641 }
642
643 fn complete_cache(cache: ReverseImageSearchCache) -> (String, String) {
644 (cache.imgbb_url, cache.google_lens_url)
645 }
646
647 #[derive(Deserialize)]
648 struct ImgBbUploadResponse {
649 success: bool,
650 data: Option<ImgBbUploadData>,
651 error: Option<ImgBbUploadError>,
652 }
653
654 #[derive(Deserialize)]
655 struct ImgBbUploadData {
656 url: Option<String>,
657 }
658
659 #[derive(Deserialize)]
660 struct ImgBbUploadError {
661 message: Option<String>,
662 }
663
664 async fn upload_image(image_path: &Path, api_key: &str) -> ThreadResult<String> {
665 let bytes = tokio::fs::read(image_path)
666 .await
667 .map_err(|error| error.to_string())?;
668 if bytes.is_empty() {
669 return Err("Image file is empty".to_string());
670 }
671 let file_name = image_path
672 .file_name()
673 .and_then(|name| name.to_str())
674 .unwrap_or("image")
675 .to_string();
676 let mime = mime_guess::from_path(image_path).first_or_octet_stream();
677 let image = reqwest::multipart::Part::bytes(bytes)
678 .file_name(file_name)
679 .mime_str(mime.essence_str())
680 .map_err(|error| error.to_string())?;
681 let response = reqwest::Client::new()
682 .post("https://api.imgbb.com/1/upload")
683 .query(&[("key", api_key)])
684 .multipart(reqwest::multipart::Form::new().part("image", image))
685 .send()
686 .await
687 .map_err(|error| error.to_string())?;
688 let status = response.status();
689 let body = response.text().await.map_err(|error| error.to_string())?;
690 if !status.is_success() {
691 return Err(format!("ImgBB upload failed with status {status}"));
692 }
693 let parsed = serde_json::from_str::<ImgBbUploadResponse>(&body)
694 .map_err(|error| error.to_string())?;
695 if !parsed.success {
696 return Err(parsed
697 .error
698 .and_then(|error| error.message)
699 .unwrap_or_else(|| "ImgBB upload was not successful".to_string()));
700 }
701 parsed
702 .data
703 .and_then(|data| data.url)
704 .ok_or_else(|| "ImgBB response did not contain an image URL".to_string())
705 }
706
707 async fn run_reverse_image_search_url(
708 thread_id: &str,
709 query: Option<&str>,
710 ) -> ThreadResult<ReverseImageSearchOutcome> {
711 let storage = active_storage()?;
712 let thread = storage
713 .load_thread(thread_id)
714 .map_err(|error| error.to_string())?;
715 let hash = thread.metadata.image_hash;
716 let cached = storage
717 .get_reverse_image_search_cache(&hash)
718 .map_err(|error| error.to_string())?;
719 let (imgbb_url, google_lens_url) = match cached.map(complete_cache) {
720 Some(cache) => cache,
721 None => {
722 let credential = tokio::task::spawn_blocking(|| {
723 let profiles = storage::profile_store().map_err(|error| error.to_string())?;
724 let profile_id = profiles
725 .get_active_profile_id()
726 .map_err(|error| error.to_string())?
727 .ok_or_else(|| {
728 if session_api_keys_active() {
729 "ImgBB is unavailable. Set IMGBB_API_KEY in the shell or repo .env."
730 .to_string()
731 } else {
732 "An active profile is required for reverse image search".to_string()
733 }
734 })?;
735 get_decrypted_api_key(&profiles, ApiKeyProvider::ImgBb, &profile_id)
736 .map_err(|error| error.to_string())?
737 .ok_or_else(|| {
738 if session_api_keys_active() {
739 "ImgBB is unavailable. Set IMGBB_API_KEY in the shell or repo .env."
740 .to_string()
741 } else {
742 "ImgBB key is not configured".to_string()
743 }
744 })
745 })
746 .await
747 .map_err(|error| error.to_string())??;
748 let image_path = storage
749 .find_object_blob(&hash)
750 .map_err(|error| error.to_string())?;
751 let imgbb_url = upload_image(&image_path, credential.api_key.expose()).await?;
752 let google_lens_url = lens_url(&imgbb_url)?;
753 storage
754 .save_reverse_image_search_cache(
755 &hash,
756 imgbb_url.clone(),
757 google_lens_url.clone(),
758 )
759 .map_err(|error| error.to_string())?;
760 (imgbb_url, google_lens_url)
761 }
762 };
763 let mut opened = Url::parse(&google_lens_url).map_err(|error| error.to_string())?;
764 if let Some(query) = query.map(str::trim).filter(|value| !value.is_empty()) {
765 opened.query_pairs_mut().append_pair("q", query);
766 }
767 Ok(ReverseImageSearchOutcome {
768 imgbb_url,
769 google_lens_url,
770 opened_url: opened.into(),
771 })
772 }
773
774 pub async fn reverse_image_search_url(
775 thread_id: &str,
776 query: Option<&str>,
777 ) -> ThreadResult<ReverseImageSearchOutcome> {
778 let (search_id, mut cancellation) = begin_reverse_search(thread_id)?;
779 let result = tokio::select! {
780 _ = cancellation.changed() => Err("Reverse image search cancelled".to_string()),
781 result = run_reverse_image_search_url(thread_id, query) => result,
782 };
783 finish_reverse_search(thread_id, search_id)?;
784 result
785 }
786
787 struct Lcg(u64);
788
789 impl Lcg {
790 #[inline]
791 fn next(&mut self) -> u64 {
792 self.0 = self
793 .0
794 .wrapping_mul(6_364_136_223_846_793_005)
795 .wrapping_add(1_442_695_040_888_963_407);
796 self.0
797 }
798
799 #[inline]
800 fn range(&mut self, lo: u32, hi: u32) -> u32 {
801 if hi <= lo + 1 {
802 return lo;
803 }
804 lo + (self.next() as u32 % (hi - lo))
805 }
806 }
807
808 pub fn detect_image_tone_from_bytes(bytes: &[u8]) -> Option<String> {
809 let img = image::load_from_memory(bytes).ok()?;
810 let (width, height) = img.dimensions();
811
812 if width == 0 || height == 0 {
813 return Some("dark".to_string());
814 }
815
816 let max_dim = 256;
817 let thumb = img.thumbnail(max_dim, max_dim);
818 let blurred = imageops::blur(&thumb, 1.5);
819
820 let (w, h) = blurred.dimensions();
821 if w == 0 || h == 0 {
822 return Some("dark".to_string());
823 }
824
825 let srgb_to_linear = |c: u8| -> f32 {
826 let f = c as f32 / 255.0;
827 if f <= 0.04045 {
828 f / 12.92
829 } else {
830 ((f + 0.055) / 1.055).powf(2.4)
831 }
832 };
833
834 let get_luminance = |r: u8, g: u8, b: u8| -> f32 {
835 0.2126 * srgb_to_linear(r) + 0.7152 * srgb_to_linear(g) + 0.0722 * srgb_to_linear(b)
836 };
837
838 let mut sum_lum = 0.0;
839 let mut count = 0;
840 for pixel in blurred.pixels() {
841 if pixel[3] > 128 {
842 sum_lum += get_luminance(pixel[0], pixel[1], pixel[2]);
843 count += 1;
844 }
845 }
846
847 if count == 0 {
848 return Some("light".to_string());
849 }
850
851 let global_mean = sum_lum / count as f32;
852 if global_mean <= 0.05 {
853 return Some("dark".to_string());
854 }
855 if global_mean >= 0.75 {
856 return Some("light".to_string());
857 }
858
859 let mut rng = Lcg(0xDEAD_BEEF_CAFE_1337);
860 let grid_size = 12;
861 let spc = 8;
862
863 let thresh = 0.179;
864 let mut dark_score = 0.0;
865 let mut light_score = 0.0;
866
867 for gy in 0..grid_size {
868 for gx in 0..grid_size {
869 let cx0 = gx * w / grid_size;
870 let cx1 = ((gx + 1) * w / grid_size).max(cx0 + 1);
871 let cy0 = gy * h / grid_size;
872 let cy1 = ((gy + 1) * h / grid_size).max(cy0 + 1);
873
874 for _ in 0..spc {
875 let x = rng.range(cx0, cx1).min(w.saturating_sub(1));
876 let y = rng.range(cy0, cy1).min(h.saturating_sub(1));
877
878 if blurred.get_pixel(x, y)[3] < 128 {
879 continue;
880 }
881
882 let mut local_dark = 0;
883 let mut local_light = 0;
884 let mut valid_neighbors = 0;
885
886 for dy in -1..=1 {
887 for dx in -1..=1 {
888 let nx = (x as i32 + dx) as u32;
889 let ny = (y as i32 + dy) as u32;
890 if nx < w && ny < h {
891 let px = blurred.get_pixel(nx, ny);
892 if px[3] > 128 {
893 let l = get_luminance(px[0], px[1], px[2]);
894 if l < thresh {
895 local_dark += 1;
896 } else {
897 local_light += 1;
898 }
899 valid_neighbors += 1;
900 }
901 }
902 }
903 }
904
905 if valid_neighbors > 0 {
906 let confidence =
907 (local_dark as f32 - local_light as f32).abs() / valid_neighbors as f32;
908 let is_node_dark = local_dark >= local_light;
909
910 if is_node_dark {
911 dark_score += 1.0 + confidence;
912 } else {
913 light_score += 1.0 + confidence;
914 }
915 }
916 }
917 }
918 }
919
920 if dark_score >= light_score {
921 Some("dark".to_string())
922 } else {
923 Some("light".to_string())
924 }
925 }
926
927 pub fn detect_image_tone_from_path(path: &Path) -> ThreadResult<Option<String>> {
928 let bytes = std::fs::read(path).map_err(|error| error.to_string())?;
929 if bytes.is_empty() {
930 return Err("Image file is empty".to_string());
931 }
932 Ok(detect_image_tone_from_bytes(&bytes))
933 }
934}
935
936pub mod ocr {
937 use serde::Serialize;
938 use squigit_ocr::models::DEFAULT_OCR_MODEL_ID;
939 use squigit_ocr::ocr::{persist_boxes_to_thread_storage, OcrRequest};
940 use std::collections::BTreeMap;
941 use std::path::PathBuf;
942 use std::sync::atomic::{AtomicU64, Ordering};
943 use std::sync::{Arc, Mutex, OnceLock};
944 use tokio::sync::Mutex as AsyncMutex;
945
946 use crate::services::{ocr, ocr_models};
947
948 use super::{active_storage, ThreadResult};
949
950 #[derive(Serialize)]
951 #[serde(rename_all = "camelCase")]
952 pub struct OcrThreadSnapshot {
953 pub thread_id: String,
954 pub thread_title: String,
955 pub image_path: String,
956 pub image_hash: String,
957 pub image_tone: Option<String>,
958 pub ocr_data: crate::storage::OcrAnnotations,
959 }
960
961 #[derive(Clone)]
962 struct OcrJobRecord {
963 job_id: String,
964 thread_id: String,
965 model_id: String,
966 status: String,
967 output: Option<String>,
968 error: Option<String>,
969 }
970
971 #[derive(Serialize)]
972 #[serde(rename_all = "camelCase")]
973 pub struct OcrJobSnapshot {
974 pub job_id: String,
975 pub thread_id: String,
976 pub model_id: String,
977 pub status: String,
978 pub has_output: bool,
979 pub error: Option<String>,
980 }
981
982 impl From<&OcrJobRecord> for OcrJobSnapshot {
983 fn from(job: &OcrJobRecord) -> Self {
984 Self {
985 job_id: job.job_id.clone(),
986 thread_id: job.thread_id.clone(),
987 model_id: job.model_id.clone(),
988 status: job.status.clone(),
989 has_output: job.output.is_some(),
990 error: job.error.clone(),
991 }
992 }
993 }
994
995 static OCR_JOBS: OnceLock<Arc<Mutex<BTreeMap<u64, OcrJobRecord>>>> = OnceLock::new();
996 static OCR_JOB_RUN_LOCK: OnceLock<AsyncMutex<()>> = OnceLock::new();
997 static NEXT_OCR_JOB_ID: AtomicU64 = AtomicU64::new(1);
998
999 fn lock_jobs(
1000 jobs: &Mutex<BTreeMap<u64, OcrJobRecord>>,
1001 ) -> ThreadResult<std::sync::MutexGuard<'_, BTreeMap<u64, OcrJobRecord>>> {
1002 jobs.lock()
1003 .map_err(|_| "OCR job queue state is unavailable".to_string())
1004 }
1005
1006 fn ocr_job_sequence(job_id: &str) -> Option<u64> {
1007 job_id.strip_prefix("ocr-")?.parse().ok()
1008 }
1009
1010 fn ocr_jobs() -> &'static Arc<Mutex<BTreeMap<u64, OcrJobRecord>>> {
1011 OCR_JOBS.get_or_init(|| Arc::new(Mutex::new(BTreeMap::new())))
1012 }
1013
1014 fn ocr_job_run_lock() -> &'static AsyncMutex<()> {
1015 OCR_JOB_RUN_LOCK.get_or_init(|| AsyncMutex::new(()))
1016 }
1017
1018 pub fn load_ocr_thread(thread_id: &str) -> ThreadResult<OcrThreadSnapshot> {
1019 let storage = active_storage()?;
1020 let thread = storage
1021 .load_thread(thread_id)
1022 .map_err(|error| error.to_string())?;
1023 let image_path = storage
1024 .get_image_path(&thread.metadata.image_hash)
1025 .map_err(|error| error.to_string())?;
1026 Ok(OcrThreadSnapshot {
1027 thread_id: thread_id.to_string(),
1028 thread_title: thread.metadata.title,
1029 image_path,
1030 image_hash: thread.metadata.image_hash,
1031 image_tone: thread.image_tone,
1032 ocr_data: thread.ocr_data,
1033 })
1034 }
1035
1036 fn sidecar_request(image_path: String, model_id: &str) -> ThreadResult<OcrRequest> {
1037 let sidecar_path = squigit_ocr::sidecar::resolve_sidecar_path();
1038 let rec_model_dir_override = (model_id != DEFAULT_OCR_MODEL_ID)
1039 .then(|| ocr_models().map(|models| models.get_model_dir(model_id)))
1040 .transpose()?;
1041 Ok(OcrRequest {
1042 sidecar_path,
1043 image_path: PathBuf::from(image_path),
1044 rec_model_dir_override,
1045 })
1046 }
1047
1048 async fn run_ocr_thread_job(thread_id: &str, model_id: &str) -> ThreadResult<String> {
1049 let storage = active_storage()?;
1050 let thread = storage
1051 .load_thread(thread_id)
1052 .map_err(|error| error.to_string())?;
1053 let image_path = storage
1054 .get_image_path(&thread.metadata.image_hash)
1055 .map_err(|error| error.to_string())?;
1056 let request = sidecar_request(image_path, model_id)?;
1057 let result = ocr()
1058 .run(request)
1059 .await
1060 .map_err(|error| error.to_string())?;
1061 persist_boxes_to_thread_storage(&storage, thread_id, model_id, &result.boxes)
1062 .map_err(|error| error.to_string())?;
1063 serde_json::to_string(
1064 &storage
1065 .get_ocr_annotations(thread_id)
1066 .map_err(|error| error.to_string())?,
1067 )
1068 .map_err(|error| error.to_string())
1069 }
1070
1071 async fn run_queued_ocr_job(sequence: u64, jobs: Arc<Mutex<BTreeMap<u64, OcrJobRecord>>>) {
1072 let _queue_guard = ocr_job_run_lock().lock().await;
1073 let request = {
1074 let Ok(mut records) = lock_jobs(&jobs) else {
1075 return;
1076 };
1077 let Some(job) = records.get_mut(&sequence) else {
1078 return;
1079 };
1080 if job.status == "cancelled" {
1081 return;
1082 }
1083 job.status = "running".to_string();
1084 (job.thread_id.clone(), job.model_id.clone())
1085 };
1086
1087 let result = run_ocr_thread_job(&request.0, &request.1).await;
1088 let Ok(mut records) = lock_jobs(&jobs) else {
1089 return;
1090 };
1091 let Some(job) = records.get_mut(&sequence) else {
1092 return;
1093 };
1094 if job.status == "cancelled" {
1095 return;
1096 }
1097 match result {
1098 Ok(output) => {
1099 job.status = "completed".to_string();
1100 job.output = Some(output);
1101 job.error = None;
1102 }
1103 Err(error) => {
1104 job.status = "failed".to_string();
1105 job.error = Some(error);
1106 }
1107 }
1108 }
1109
1110 fn spawn_ocr_job(sequence: u64, jobs: Arc<Mutex<BTreeMap<u64, OcrJobRecord>>>) {
1111 let worker_jobs = Arc::clone(&jobs);
1112 let worker = tokio::spawn(async move {
1113 run_queued_ocr_job(sequence, worker_jobs).await;
1114 });
1115 tokio::spawn(async move {
1116 let Err(error) = worker.await else {
1117 return;
1118 };
1119 if let Ok(mut records) = lock_jobs(&jobs) {
1120 if let Some(job) = records.get_mut(&sequence) {
1121 if job.status != "cancelled" {
1122 job.status = "failed".to_string();
1123 job.error = Some(format!("OCR job worker crashed: {error}"));
1124 }
1125 }
1126 }
1127 });
1128 }
1129
1130 pub fn start_ocr_thread_job(thread_id: &str, model_id: &str) -> ThreadResult<String> {
1131 let jobs = ocr_jobs();
1132 {
1133 let records = lock_jobs(jobs)?;
1134 if let Some(existing) = records.values().rev().find(|job| {
1135 job.thread_id == thread_id
1136 && job.model_id == model_id
1137 && matches!(job.status.as_str(), "queued" | "running")
1138 }) {
1139 return Ok(existing.job_id.clone());
1140 }
1141 }
1142
1143 let sequence = NEXT_OCR_JOB_ID.fetch_add(1, Ordering::Relaxed);
1144 let job_id = format!("ocr-{sequence}");
1145 {
1146 let mut records = lock_jobs(jobs)?;
1147 records.insert(
1148 sequence,
1149 OcrJobRecord {
1150 job_id: job_id.clone(),
1151 thread_id: thread_id.to_string(),
1152 model_id: model_id.to_string(),
1153 status: "queued".to_string(),
1154 output: None,
1155 error: None,
1156 },
1157 );
1158 }
1159 spawn_ocr_job(sequence, Arc::clone(jobs));
1160 Ok(job_id)
1161 }
1162
1163 pub fn get_ocr_jobs_snapshot() -> ThreadResult<Vec<OcrJobSnapshot>> {
1164 if let Some(jobs) = OCR_JOBS.get() {
1165 Ok(lock_jobs(jobs)?
1166 .values()
1167 .map(OcrJobSnapshot::from)
1168 .collect())
1169 } else {
1170 Ok(Vec::new())
1171 }
1172 }
1173
1174 pub fn get_ocr_job_output(job_id: &str) -> ThreadResult<Option<String>> {
1175 let Some(jobs) = OCR_JOBS.get() else {
1176 return Ok(None);
1177 };
1178 let Some(sequence) = ocr_job_sequence(job_id) else {
1179 return Ok(None);
1180 };
1181 Ok(lock_jobs(jobs)?
1182 .get(&sequence)
1183 .and_then(|job| job.output.clone()))
1184 }
1185
1186 pub async fn cancel_ocr_job(job_id: &str) -> ThreadResult<()> {
1187 let Some(jobs) = OCR_JOBS.get() else {
1188 return Ok(());
1189 };
1190 let Some(sequence) = ocr_job_sequence(job_id) else {
1191 return Ok(());
1192 };
1193 let was_running = {
1194 let mut records = lock_jobs(jobs)?;
1195 let Some(job) = records.get_mut(&sequence) else {
1196 return Ok(());
1197 };
1198 if matches!(job.status.as_str(), "completed" | "failed" | "cancelled") {
1199 return Ok(());
1200 }
1201 let was_running = job.status == "running";
1202 job.status = "cancelled".to_string();
1203 was_running
1204 };
1205 if was_running {
1206 ocr()
1207 .cancel_current_job()
1208 .await
1209 .map_err(|error| error.to_string())?;
1210 }
1211 Ok(())
1212 }
1213}