use iced::advanced::{image as advanced_image, text};
use iced::widget::{image, markdown};
use iced::{ContentFit, Element, Font, Length};
use std::collections::{HashMap, VecDeque};
use std::sync::{LazyLock, Mutex};
use crate::util::media_target;
use crate::util::{
MEDIA_MARKER_RE, MediaMarkerKind, UnwrapPoison, file_name_or_path, parse_media_marker,
};
use ::image as image_crate;
pub(crate) fn preprocess(content: &str) -> String {
let s = replace_transcription(content);
let s = replace_media_markers(&s);
replace_saved_annotations(&s)
}
fn replace_transcription(s: &str) -> String {
static RE: LazyLock<regex::Regex> = LazyLock::new(|| {
regex::Regex::new(r"\[(Audio|Video) transcription of [^\]]+\]:\s*")
.expect("transcription regex must compile")
});
RE.replace_all(s, |caps: ®ex::Captures| {
match caps.get(1).map(|m| m.as_str()) {
Some("Audio") => "🔊 ",
_ => "🎬 ",
}
})
.to_string()
}
fn replace_media_markers(s: &str) -> String {
MEDIA_MARKER_RE
.replace_all(s, |caps: ®ex::Captures| {
let (kind, path) = parse_media_marker(caps);
match kind {
MediaMarkerKind::Image => {
let p = path.trim();
if p.starts_with(crate::util::DATA_URI_OMITTED_PREFIX) {
"🖼️ image".to_string()
} else if p.starts_with("data:") {
format!("")
} else if media_target::is_valid_remote_url(p)
|| std::path::Path::new(p).is_absolute()
{
format!("")
} else {
caps.get_match().as_str().to_string()
}
}
MediaMarkerKind::Audio => format!("🎵 {}", file_name_or_path(path)),
MediaMarkerKind::File => {
if path.trim().starts_with("data:") {
"📄 file".to_string()
} else {
format!("📄 {}", file_name_or_path(path))
}
}
MediaMarkerKind::Video => format!("🎬 Video: {}", file_name_or_path(path)),
}
})
.to_string()
}
fn replace_saved_annotations(s: &str) -> String {
static SAVED_IMAGE_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
regex::Regex::new(r"\n*\[Saved image: [^\]]*\]")
.expect("saved image annotation regex must compile")
});
static SAVED_VIDEO_RE: LazyLock<regex::Regex> = LazyLock::new(|| {
regex::Regex::new(r"\[Saved video: ([^\]]*)\]")
.expect("saved video annotation regex must compile")
});
let s = SAVED_IMAGE_RE.replace_all(s, "").to_string();
SAVED_VIDEO_RE
.replace_all(&s, |caps: ®ex::Captures| {
let path = caps.get(1).map_or("", |m| m.as_str());
format!("🎬 Video: {}", file_name_or_path(path))
})
.to_string()
}
const MAX_IMAGE_LONGEST_SIDE_PX: u32 = 2048;
fn downscale_raster(width: u32, height: u32, rgba: Vec<u8>) -> Option<(u32, u32, Vec<u8>)> {
if width <= MAX_IMAGE_LONGEST_SIDE_PX && height <= MAX_IMAGE_LONGEST_SIDE_PX {
return Some((width, height, rgba));
}
let img = image_crate::DynamicImage::ImageRgba8(image_crate::RgbaImage::from_raw(
width, height, rgba,
)?);
let resized = img
.thumbnail(MAX_IMAGE_LONGEST_SIDE_PX, MAX_IMAGE_LONGEST_SIDE_PX)
.into_rgba8();
Some((resized.width(), resized.height(), resized.into_raw()))
}
#[must_use]
fn render_raster(bytes: &[u8]) -> Option<(u32, u32, Vec<u8>)> {
let (width, height, rgba) =
media_target::decode_raster_bytes(bytes, media_target::raster_decode_limits())?;
downscale_raster(width, height, rgba)
}
#[must_use]
fn decode_data_uri_image(uri: &str) -> Option<(u32, u32, Vec<u8>)> {
let (width, height, rgba) =
media_target::decode_native_data_uri(uri, media_target::raster_decode_limits())?;
downscale_raster(width, height, rgba)
}
#[derive(Debug)]
enum CachedDataUriImage {
Decoded(image::Handle),
Failed,
}
static DATA_URI_IMAGE_CACHE: LazyLock<Mutex<BoundedLruCache<String, CachedDataUriImage>>> =
LazyLock::new(|| Mutex::new(BoundedLruCache::with_capacity(MAX_CACHED_IMAGES)));
struct BoundedLruCache<K, V> {
entries: HashMap<K, V>,
lru: VecDeque<K>,
cap: usize,
}
impl<K, V> BoundedLruCache<K, V>
where
K: std::hash::Hash + Eq + Clone,
{
fn with_capacity(cap: usize) -> Self {
Self {
entries: HashMap::new(),
lru: VecDeque::new(),
cap,
}
}
fn get<Q>(&mut self, key: &Q) -> Option<&V>
where
K: std::borrow::Borrow<Q>,
Q: std::hash::Hash + Eq + ?Sized,
{
let hit = self.entries.get(key);
if hit.is_some() {
if let Some(pos) = self.lru.iter().position(|k| k.borrow() == key) {
let key = self.lru.remove(pos).expect("position found by iteration");
self.lru.push_back(key);
}
}
hit
}
fn insert(&mut self, key: K, value: V) {
if let Some(pos) = self.lru.iter().position(|k| k == &key) {
self.lru.remove(pos);
}
self.entries.insert(key.clone(), value);
self.lru.push_back(key);
while self.entries.len() > self.cap {
let oldest = self
.lru
.pop_front()
.expect("lru mirrors entries, so a front key always exists");
self.entries.remove(&oldest);
}
}
}
#[must_use]
fn data_uri_image_handle(uri: &str) -> Option<image::Handle> {
media_target::data_uri_base64_payload(uri)?;
{
let mut cache = DATA_URI_IMAGE_CACHE.lock().unwrap_poison();
match cache.get(uri) {
Some(CachedDataUriImage::Decoded(handle)) => return Some(handle.clone()),
Some(CachedDataUriImage::Failed) => return None,
None => {}
}
}
let outcome = match decode_data_uri_image(uri) {
Some((width, height, rgba)) => {
CachedDataUriImage::Decoded(image::Handle::from_rgba(width, height, rgba))
}
None => CachedDataUriImage::Failed,
};
let handle = match &outcome {
CachedDataUriImage::Decoded(handle) => Some(handle.clone()),
CachedDataUriImage::Failed => None,
};
DATA_URI_IMAGE_CACHE
.lock()
.unwrap_poison()
.insert(uri.to_string(), outcome);
handle
}
const MAX_CACHED_IMAGES: usize = 32;
#[derive(Debug)]
enum CachedLocalImage {
Decoded(image::Handle),
Failed,
}
static LOCAL_IMAGE_CACHE: LazyLock<
Mutex<BoundedLruCache<(std::path::PathBuf, std::time::SystemTime), CachedLocalImage>>,
> = LazyLock::new(|| Mutex::new(BoundedLruCache::with_capacity(MAX_CACHED_IMAGES)));
#[must_use]
fn local_path_image_handle(path: &str) -> Option<image::Handle> {
let p = std::path::Path::new(path);
if !p.is_absolute() {
return None;
}
let meta = std::fs::metadata(p).ok()?;
if !meta.is_file() || meta.len() > crate::util::INBOUND_IMAGE_MAX_INPUT_BYTES {
return None;
}
let mtime = meta.modified().ok()?;
let canonical = std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf());
let key = (canonical.clone(), mtime);
{
let mut cache = LOCAL_IMAGE_CACHE.lock().unwrap_poison();
match cache.get(&key) {
Some(CachedLocalImage::Decoded(handle)) => return Some(handle.clone()),
Some(CachedLocalImage::Failed) => return None,
None => {}
}
}
let outcome = match std::fs::read(&canonical)
.ok()
.and_then(|b| render_raster(&b))
{
Some((width, height, rgba)) => {
CachedLocalImage::Decoded(image::Handle::from_rgba(width, height, rgba))
}
None => CachedLocalImage::Failed,
};
let handle = match &outcome {
CachedLocalImage::Decoded(handle) => Some(handle.clone()),
CachedLocalImage::Failed => None,
};
LOCAL_IMAGE_CACHE
.lock()
.unwrap_poison()
.insert(key, outcome);
handle
}
pub(crate) struct SelectableMediaViewer;
pub(crate) fn selectable_markdown_view(
items: &[markdown::Item],
settings: impl Into<markdown::Settings>,
) -> Element<'_, markdown::Uri, iced::Theme, iced::Renderer> {
markdown::view_with(items, settings, &SelectableMediaViewer)
}
impl<'a, Theme, Renderer> markdown::Viewer<'a, markdown::Uri, Theme, Renderer>
for SelectableMediaViewer
where
Theme: markdown::Catalog + iced_selection::text::Catalog + 'a,
Renderer: text::Renderer<Paragraph = iced::advanced::graphics::text::Paragraph, Font = Font>
+ advanced_image::Renderer<Handle = image::Handle>
+ 'a,
{
fn on_link_click(url: markdown::Uri) -> markdown::Uri {
url
}
fn image(
&self,
settings: markdown::Settings,
url: &'a markdown::Uri,
_title: &'a str,
_alt: &markdown::Text,
) -> Element<'a, markdown::Uri, Theme, Renderer> {
let url_str = url.as_str();
if url_str.starts_with("data:") {
return match data_uri_image_handle(url_str) {
Some(handle) => image::Image::new(handle)
.width(Length::Fill)
.content_fit(ContentFit::Contain)
.into(),
None => iced::widget::text("🖼️ image")
.size(settings.text_size)
.into(),
};
}
if let Some(handle) = local_path_image_handle(url_str) {
image::Image::new(handle)
.width(Length::Fill)
.content_fit(ContentFit::Contain)
.into()
} else {
let filename = file_name_or_path(url_str);
iced::widget::text(format!("🖼️ {filename}"))
.size(settings.text_size)
.into()
}
}
fn heading(
&self,
settings: markdown::Settings,
level: &'a markdown::HeadingLevel,
text: &'a markdown::Text,
index: usize,
) -> Element<'a, markdown::Uri, Theme, Renderer> {
iced_selection::markdown::heading(settings, level, text, index, |url| url)
}
fn paragraph(
&self,
settings: markdown::Settings,
text: &markdown::Text,
) -> Element<'a, markdown::Uri, Theme, Renderer> {
iced_selection::markdown::paragraph(settings, text, |url| url)
}
fn code_block(
&self,
settings: markdown::Settings,
_language: Option<&'a str>,
_code: &'a str,
lines: &'a [markdown::Text],
) -> Element<'a, markdown::Uri, Theme, Renderer> {
iced_selection::markdown::code_block(settings, lines, |url| url)
}
fn unordered_list(
&self,
settings: markdown::Settings,
bullets: &'a [markdown::Bullet],
) -> Element<'a, markdown::Uri, Theme, Renderer> {
iced_selection::markdown::unordered_list(self, settings, bullets)
}
fn ordered_list(
&self,
settings: markdown::Settings,
start: u64,
bullets: &'a [markdown::Bullet],
) -> Element<'a, markdown::Uri, Theme, Renderer> {
iced_selection::markdown::ordered_list(self, settings, start, bullets)
}
}
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use std::io::Cursor;
#[expect(clippy::too_many_lines)] #[test]
fn preprocess_table() {
let tmp = tempfile::tempdir().unwrap();
let photo = tmp.path().join("photo.png");
let spaced = tmp.path().join("my file.png");
std::fs::write(&photo, tiny_png(2, 1)).unwrap();
std::fs::write(&spaced, tiny_png(2, 1)).unwrap();
let photo_str = photo.to_string_lossy().into_owned();
let spaced_str = spaced.to_string_lossy().into_owned();
let valid_uri = data_uri(&tiny_png(2, 1));
let cleaned = tmp.path().join("cleaned_up.png"); let cleaned_str = cleaned.to_string_lossy().into_owned();
let cases: Vec<(&str, String, String)> = vec![
(
"replaces an image marker",
format!("Look [IMAGE:{photo_str}] here"),
format!("Look  here"),
),
(
"replaces an image marker with spaces in the path",
format!("img [IMAGE:{spaced_str}]"),
format!("img "),
),
(
"renders a cleaned-up absolute temp-file marker as an image item (placeholder in the viewer)",
format!("Look [IMAGE:{cleaned_str}] here"),
format!("Look  here"),
),
(
"renders the omitted-data-URI placeholder as the image affordance, not raw marker text",
format!(
"Look [IMAGE:{} (35 bytes)>] here",
crate::util::DATA_URI_OMITTED_PREFIX
),
"Look 🖼️ image here".into(),
),
(
"replaces an audio marker",
"Listen [AUDIO:/tmp/recording.ogg]".into(),
"Listen 🎵 recording.ogg".into(),
),
(
"replaces an audio marker in a nested path",
"hear [AUDIO:/dir/subdir/rec.ogg]".into(),
"hear 🎵 rec.ogg".into(),
),
(
"replaces a video marker",
"Watch [VIDEO:/tmp/video.mp4]".into(),
"Watch 🎬 Video: video.mp4".into(),
),
(
"replaces a file marker",
"Get [FILE:/tmp/report.pdf]".into(),
"Get 📄 report.pdf".into(),
),
(
"names a data-URI file marker generically instead of dumping its payload",
"Get [FILE:data:text/plain;base64,aGVsbG8=]".into(),
"Get 📄 file".into(),
),
(
"replaces an audio transcription block",
"[Audio transcription of recording.ogg]: Hello world".into(),
"🔊 Hello world".into(),
),
(
"renders every line of a multiline audio transcription under 🔊",
"[Audio transcription of voice.ogg]: Line one\nLine two".into(),
"🔊 Line one\nLine two".into(),
),
(
"renders every line of a multiline video transcription under 🎬 and leaves following message content untouched",
"[Video transcription of clip.mp4]: Line one\nLine two\n\nEdit this".into(),
"🎬 Line one\nLine two\n\nEdit this".into(),
),
(
"video-transcription format contains Video and is handled before the [VIDEO:...] pattern",
"[Video transcription of clip.mp4]: hi there [VIDEO:/tmp/other.mp4]".into(),
"🎬 hi there 🎬 Video: other.mp4".into(),
),
(
"audio-transcription format contains Audio and is handled before the [AUDIO:...] pattern",
"[Audio transcription of msg.ogg]: hi there [AUDIO:/tmp/other.ogg]".into(),
"🔊 hi there 🎵 other.ogg".into(),
),
(
"replaces mixed image/audio/video markers",
format!("![]() [IMAGE:{photo_str}] and [AUDIO:/tmp/b.ogg] end [VIDEO:/tmp/c.mp4]"),
format!("![]()  and 🎵 b.ogg end 🎬 Video: c.mp4"),
),
(
"leaves content without markers unchanged",
"Hello world".into(),
"Hello world".into(),
),
(
"returns the empty string unchanged",
String::new(),
String::new(),
),
(
"strips a saved image annotation and its blank-line separator",
format!("[IMAGE:{valid_uri}]\n\n[Saved image: /uploads/img.png]"),
format!(""),
),
(
"strips a saved image annotation at the start with no preceding separator",
"[Saved image: /uploads/img.png]".into(),
String::new(),
),
(
"turns a saved video annotation into a visible clip placeholder",
"Clip [Saved video: /uploads/clip.mp4] here".into(),
"Clip 🎬 Video: clip.mp4 here".into(),
),
(
"keeps the saved video clip visible alongside its transcription",
"[Video transcription of clip.mp4]: Line one\n\n[Saved video: /uploads/clip.mp4]"
.into(),
"🎬 Line one\n\n🎬 Video: clip.mp4".into(),
),
];
for (i, (name, input, expected)) in cases.iter().enumerate() {
assert_eq!(preprocess(input), *expected, "case {i} ({name})");
}
}
#[test]
fn file_name_or_path_table() {
let cases: &[(&str, &str, &str)] = &[
("unix path", "/foo/bar.txt", "bar.txt"),
("nested path", "/foo/bar/doc.txt", "doc.txt"),
("no directory component", "bar.txt", "bar.txt"),
("trailing slash is normalized on Unix", "/foo/bar/", "bar"),
];
for (i, (name, input, expected)) in cases.iter().enumerate() {
assert_eq!(file_name_or_path(input), *expected, "case {i} ({name})");
}
}
fn tiny_png(width: u32, height: u32) -> Vec<u8> {
let img = ::image::RgbaImage::from_pixel(width, height, ::image::Rgba([255, 0, 0, 255]));
let mut buf = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ::image::ImageFormat::Png)
.expect("test PNG must encode");
buf
}
fn data_uri(png: &[u8]) -> String {
format!("data:image/png;base64,{}", STANDARD.encode(png))
}
#[test]
fn decode_data_uri_image_table() {
let valid_uri = data_uri(&tiny_png(2, 1));
let invalid_base64 = "data:image/png;base64,%%%not-base64%%%".to_string();
let non_image = format!("data:text/plain;base64,{}", STANDARD.encode(b"hello world"));
let oversized_payload = format!(
"data:image/png;base64,{}",
"A".repeat(media_target::MAX_DATA_URI_ENCODED_BYTES + 1)
);
let oversized_dimensions = data_uri(&tiny_png(MAX_IMAGE_LONGEST_SIDE_PX + 1, 1));
let missing_marker = "data:image/png,AAAA".to_string();
let mut jbuf = Vec::new();
::image::RgbImage::from_pixel(1, 1, ::image::Rgb([255, 0, 0]))
.write_to(&mut Cursor::new(&mut jbuf), ::image::ImageFormat::Jpeg)
.expect("test JPEG must encode");
let mismatched_uri = format!("data:image/png;base64,{}", STANDARD.encode(&jbuf));
#[expect(clippy::type_complexity)] let cases: &[(&str, String, Option<(u32, u32, Vec<u8>)>)] = &[
(
"decodes a solid-red 2×1 PNG",
valid_uri,
Some((2, 1, vec![255, 0, 0, 255, 255, 0, 0, 255])),
),
("rejects invalid base64", invalid_base64, None),
("rejects non-image bytes", non_image, None),
(
"rejects an oversized payload by a pure length check before any base64 work",
oversized_payload,
None,
),
(
"downscales a valid but over-display-cap raster to the render cap",
oversized_dimensions,
Some((
MAX_IMAGE_LONGEST_SIDE_PX,
1,
[255, 0, 0, 255].repeat(MAX_IMAGE_LONGEST_SIDE_PX as usize),
)),
),
(
"rejects a data URI missing the base64 marker",
missing_marker,
None,
),
(
"rejects a data URI whose declared subtype mismatches the actual bytes",
mismatched_uri,
None,
),
];
for (i, (name, uri, expected)) in cases.iter().enumerate() {
assert_eq!(
decode_data_uri_image(uri).as_ref(),
expected.as_ref(),
"case {i} ({name})"
);
}
}
#[test]
fn data_uri_image_handle_caches_stable_handle() {
let uri = data_uri(&tiny_png(3, 2));
let first = data_uri_image_handle(&uri).expect("valid data URI decodes");
let second = data_uri_image_handle(&uri).expect("cached hit still decodes");
assert_eq!(first.id(), second.id());
}
#[test]
fn data_uri_image_handle_caches_failures() {
let uri = "data:image/png;base64,%%%not-base64%%%";
assert_eq!(data_uri_image_handle(uri), None);
{
let cache = DATA_URI_IMAGE_CACHE.lock().unwrap_poison();
assert!(
matches!(cache.entries.get(uri), Some(CachedDataUriImage::Failed)),
"failed decode must be cached, got {:?}",
cache.entries.get(uri)
);
}
assert_eq!(data_uri_image_handle(uri), None);
}
#[test]
fn data_uri_image_handle_rejects_oversized_payload_without_caching() {
let huge = "A".repeat(media_target::MAX_DATA_URI_ENCODED_BYTES + 1);
let uri = format!("data:image/png;base64,{huge}");
assert_eq!(data_uri_image_handle(&uri), None);
let cache = DATA_URI_IMAGE_CACHE.lock().unwrap_poison();
assert!(
!cache.entries.contains_key(&uri),
"over-cap payload must not be cached"
);
}
#[test]
fn data_uri_marker_survives_markdown_parse() {
let uri = data_uri(&tiny_png(2, 1));
let processed = preprocess(&format!("[IMAGE:{uri}]"));
let items: Vec<_> = markdown::parse(&processed).collect();
let [markdown::Item::Image { url, .. }] = items.as_slice() else {
panic!("expected a single Image item, got {items:?}");
};
assert_eq!(url.as_str(), uri);
}
#[test]
fn bounded_cache_duplicate_insert_keeps_eviction_exact() {
let mut cache = BoundedLruCache::with_capacity(2);
cache.insert(1, "a");
cache.insert(1, "a2"); cache.insert(2, "b");
assert_eq!(cache.lru.len(), 2, "duplicate key must not survive in lru");
assert_eq!(cache.entries.len(), 2);
assert_eq!(cache.get(&2), Some(&"b"));
cache.insert(3, "c");
assert_eq!(cache.lru.len(), 2, "eviction must stay exact");
assert_eq!(cache.get(&1), None, "oldest (1) must be evicted");
assert_eq!(cache.get(&2), Some(&"b"));
assert_eq!(cache.get(&3), Some(&"c"));
}
}