csp_format/
clip.rs

1//! A .clip file: container + database + external chunks, with typed access
2//! and lossless saving.
3//!
4//! Saving keeps everything that was not changed: when neither the database
5//! nor any external chunk was touched, the saved bytes equal the input.
6
7use std::collections::HashMap;
8use std::path::Path;
9
10use rusqlite::Connection;
11
12use crate::container::{Container, ContainerWriter, FILE_ID_LENGTH};
13use crate::database::{self, Row, select, select_all, value_as_text};
14use crate::error::{Error, Result, bail};
15use crate::image::{GrayImage, RgbaImage, decode_png};
16use crate::model::{Canvas, Layer, LayerTree, Project};
17use crate::offscreen::Offscreen;
18use crate::schema::Schema;
19
20/// One level of a mipmap chain.
21#[derive(Debug, Clone, PartialEq, serde::Serialize)]
22pub struct MipmapLevel {
23    /// `MainId` of the `MipmapInfo` row.
24    pub info_id: i64,
25    /// `ThisScale` in percent (100, 50, 25, …).
26    pub scale: f64,
27    /// `MainId` of the `Offscreen` row holding the pixels of this level.
28    pub offscreen: i64,
29}
30
31/// A `Mipmap` row and its `MipmapInfo` levels (`BaseMipmapInfo` → `NextIndex`).
32#[derive(Debug, Clone, PartialEq, serde::Serialize)]
33pub struct Mipmap {
34    /// `MainId` of the `Mipmap` row.
35    pub main_id: i64,
36    /// The layer the mipmap belongs to (`LayerId`).
37    pub layer_id: i64,
38    /// Number of levels CSP stored (`MipmapCount`).
39    pub count: i64,
40    /// The levels, largest first.
41    pub levels: Vec<MipmapLevel>,
42}
43
44#[derive(Debug, Clone)]
45enum External {
46    /// Payload range in the original container.
47    Original(std::ops::Range<usize>),
48    New(Vec<u8>),
49}
50
51/// An opened .clip file.
52///
53/// ```no_run
54/// use csp_format::ClipFile;
55///
56/// let mut file = ClipFile::open("page0001.clip")?;
57/// println!("{} x {} px", file.canvas.width_px, file.canvas.height_px);
58/// for (layer, depth) in file.layers.flatten() {
59///     println!("{}{} ({})", "  ".repeat(depth), layer.name, layer.kind.name());
60/// }
61/// // Arbitrary columns are in the embedded SQLite database.
62/// let rows = csp_format::database::select_all(file.database(), "Layer")?;
63/// assert_eq!(rows.len(), file.layers.layers.len());
64/// file.reload()?;
65/// file.save("copy.clip")?; // byte-identical while nothing was changed
66/// # Ok::<(), csp_format::Error>(())
67/// ```
68pub struct ClipFile {
69    container: Container,
70    database: Connection,
71    changes_at_open: u64,
72    externals: Vec<(String, External)>,
73    externals_changed: bool,
74    /// Replaces the container's file id on save (see [`ClipFile::set_file_id`]).
75    file_id: Option<[u8; FILE_ID_LENGTH]>,
76    /// The `Project` row.
77    pub project: Project,
78    /// The canvas settings.
79    pub canvas: Canvas,
80    /// The layer tree, read at open (call [`ClipFile::reload`] after changing `Layer` rows).
81    pub layers: LayerTree,
82}
83
84impl ClipFile {
85    /// Reads and parses a file. The file is not kept open or changed.
86    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
87        Self::from_container(Container::read(path)?)
88    }
89
90    /// Parses the bytes of a .clip.
91    pub fn from_bytes(data: Vec<u8>) -> Result<Self> {
92        Self::from_container(Container::parse(data)?)
93    }
94
95    /// Opens the database of a parsed container and reads the typed views.
96    pub fn from_container(container: Container) -> Result<Self> {
97        let database = database::open_bytes(container.sqlite()?)?;
98        let changes_at_open = database.total_changes();
99        let externals = container
100            .external_chunks()
101            .map(|chunk| {
102                (
103                    chunk.external_id.clone().unwrap_or_default(),
104                    External::Original(chunk.payload_offset..chunk.body.end),
105                )
106            })
107            .collect();
108        let project = select_all(&database, "Project")?
109            .into_iter()
110            .next()
111            .map(Project::from_row)
112            .ok_or_else(|| Error::format("No Project row"))?;
113        let canvas = select_all(&database, "Canvas")?
114            .into_iter()
115            .next()
116            .map(Canvas::from_row)
117            .ok_or_else(|| Error::format("No Canvas row"))?;
118        let layers = LayerTree::read(&database, canvas.root_folder)?;
119        Ok(ClipFile {
120            container,
121            database,
122            changes_at_open,
123            externals,
124            externals_changed: false,
125            file_id: None,
126            project,
127            canvas,
128            layers,
129        })
130    }
131
132    /// The parsed container (chunks of the file as opened).
133    pub fn container(&self) -> &Container {
134        &self.container
135    }
136
137    /// The embedded database (in memory; changes are saved by [`ClipFile::to_bytes`]).
138    pub fn database(&self) -> &Connection {
139        &self.database
140    }
141
142    /// The parameter scheme registry (`ParamScheme` / `ElemScheme`).
143    pub fn schema(&self) -> Result<Schema> {
144        Schema::read(&self.database)
145    }
146
147    /// Which CSP generation saved the file (see [`crate::version`]).
148    pub fn version_info(&self) -> Result<crate::VersionInfo> {
149        crate::VersionInfo::read(&self.database, self.container.sqlite().ok())
150    }
151
152    /// Re-read the typed views after changing the database.
153    pub fn reload(&mut self) -> Result<()> {
154        self.project = select_all(&self.database, "Project")?
155            .into_iter()
156            .next()
157            .map(Project::from_row)
158            .ok_or_else(|| Error::format("No Project row"))?;
159        self.canvas = select_all(&self.database, "Canvas")?
160            .into_iter()
161            .next()
162            .map(Canvas::from_row)
163            .ok_or_else(|| Error::format("No Canvas row"))?;
164        self.layers = LayerTree::read(&self.database, self.canvas.root_folder)?;
165        Ok(())
166    }
167
168    /// Payload of an external chunk, None when the id is not stored.
169    pub fn external(&self, id: &str) -> Option<&[u8]> {
170        self.externals.iter().find(|(key, _)| key == id).map(|(_, data)| match data {
171            External::Original(range) => &self.container.bytes()[range.clone()],
172            External::New(bytes) => bytes.as_slice(),
173        })
174    }
175
176    /// Ids of the external chunks, in file order.
177    pub fn external_ids(&self) -> impl Iterator<Item = &str> {
178        self.externals.iter().map(|(id, _)| id.as_str())
179    }
180
181    /// Store (or replace) an external chunk's payload.
182    pub fn set_external(&mut self, id: &str, payload: Vec<u8>) {
183        self.externals_changed = true;
184        match self.externals.iter_mut().find(|(key, _)| key == id) {
185            Some(entry) => entry.1 = External::New(payload),
186            None => self.externals.push((id.to_owned(), External::New(payload))),
187        }
188    }
189
190    /// Removes an external chunk. Returns whether it was there.
191    pub fn remove_external(&mut self, id: &str) -> bool {
192        let before = self.externals.len();
193        self.externals.retain(|(key, _)| key != id);
194        let removed = self.externals.len() != before;
195        self.externals_changed |= removed;
196        removed
197    }
198
199    /// One row by `MainId`.
200    pub fn row(&self, table: &str, main_id: i64) -> Result<Option<Row>> {
201        let sql = format!("SELECT * FROM {} WHERE MainId = ?1", database::quote(table));
202        Ok(select(&self.database, &sql, [main_id])?.into_iter().next())
203    }
204
205    /// A mipmap and its levels.
206    pub fn mipmap(&self, main_id: i64) -> Result<Option<Mipmap>> {
207        let Some(row) = self.row("Mipmap", main_id)? else { return Ok(None) };
208        let mut levels = Vec::new();
209        let mut next = row.int("BaseMipmapInfo");
210        while next != 0 {
211            if levels.len() > 64 {
212                bail!("Mipmap {main_id} has a looping level list");
213            }
214            let info = self
215                .row("MipmapInfo", next)?
216                .ok_or_else(|| Error::format(format!("MipmapInfo {next} does not exist")))?;
217            levels.push(MipmapLevel { info_id: next, scale: info.real("ThisScale"), offscreen: info.int("Offscreen") });
218            next = info.int("NextIndex");
219        }
220        Ok(Some(Mipmap { main_id, layer_id: row.int("LayerId"), count: row.int("MipmapCount"), levels }))
221    }
222
223    /// Parse an `Offscreen` row and its block data (if stored).
224    pub fn offscreen(&self, main_id: i64) -> Result<Option<Offscreen>> {
225        let Some(row) = self.row("Offscreen", main_id)? else { return Ok(None) };
226        let attribute =
227            row.blob("Attribute").ok_or_else(|| Error::format(format!("Offscreen {main_id} has no Attribute")))?;
228        let payload = row.get("BlockData").and_then(value_as_text).and_then(|id| self.external(&id));
229        Offscreen::parse(attribute, payload).map(Some)
230    }
231
232    /// The full-size offscreen of a mipmap (its first level).
233    pub fn mipmap_base(&self, mipmap_id: i64) -> Result<Option<Offscreen>> {
234        let Some(mipmap) = self.mipmap(mipmap_id)? else { return Ok(None) };
235        match mipmap.levels.first() {
236            Some(level) => self.offscreen(level.offscreen),
237            None => Ok(None),
238        }
239    }
240
241    /// The stored pixels of a layer and where their top-left lies on the
242    /// canvas. None when the layer stores no pixels (CSP renders vector,
243    /// text and similar layers itself when opening the file).
244    pub fn layer_pixels(&self, layer: &Layer) -> Result<Option<(RgbaImage, (i64, i64))>> {
245        if layer.render_mipmap == 0 {
246            return Ok(None);
247        }
248        let Some(offscreen) = self.mipmap_base(layer.render_mipmap)? else { return Ok(None) };
249        if offscreen.data.is_none() {
250            return Ok(None);
251        }
252        Ok(Some((offscreen.to_rgba()?, layer.pixel_origin())))
253    }
254
255    /// The layer's pixels inside the canvas box `(x0, y0, x1, y1)`,
256    /// transparent where none are stored. Only the blocks the box overlaps
257    /// are decoded.
258    pub fn layer_pixels_in_box(&self, layer: &Layer, region: (i64, i64, i64, i64)) -> Result<RgbaImage> {
259        let (x0, y0, x1, y1) = region;
260        let base = if layer.render_mipmap == 0 { None } else { self.mipmap_base(layer.render_mipmap)? };
261        match base {
262            Some(offscreen) => {
263                let (ox, oy) = layer.pixel_origin();
264                offscreen.to_rgba_region((x0 - ox, y0 - oy, x1 - ox, y1 - oy))
265            }
266            None => Ok(RgbaImage::new((x1 - x0).max(0) as u32, (y1 - y0).max(0) as u32)),
267        }
268    }
269
270    /// Canvas boxes `(x0, y0, x1, y1)` of the 256-pixel blocks of the layer
271    /// that store data, found without decompressing anything.
272    pub fn stored_pixel_boxes(&self, layer: &Layer) -> Result<Vec<(i64, i64, i64, i64)>> {
273        if layer.render_mipmap == 0 {
274            return Ok(Vec::new());
275        }
276        let Some(offscreen) = self.mipmap_base(layer.render_mipmap)? else { return Ok(Vec::new()) };
277        let (ox, oy) = layer.pixel_origin();
278        Ok(offscreen
279            .stored_boxes()
280            .into_iter()
281            .map(|(x0, y0, x1, y1)| (x0 as i64 + ox, y0 as i64 + oy, x1 as i64 + ox, y1 as i64 + oy))
282            .collect())
283    }
284
285    /// The layer mask (0 = hidden, 255 = shown) and its canvas position.
286    pub fn layer_mask(&self, layer: &Layer) -> Result<Option<(GrayImage, (i64, i64))>> {
287        if layer.mask_mipmap == 0 {
288            return Ok(None);
289        }
290        let Some(offscreen) = self.mipmap_base(layer.mask_mipmap)? else { return Ok(None) };
291        if offscreen.data.is_none() {
292            return Ok(None);
293        }
294        Ok(Some((offscreen.to_alpha()?, layer.mask_origin())))
295    }
296
297    /// The layer palette thumbnail (`Layer.LayerRenderThumbnail` →
298    /// `LayerThumbnail.ThumbnailOffscreen`). None when there is none or it
299    /// stores no pixels.
300    pub fn layer_thumbnail(&self, layer: &Layer) -> Result<Option<RgbaImage>> {
301        let id = layer.row.int("LayerRenderThumbnail");
302        if id == 0 {
303            return Ok(None);
304        }
305        let Some(row) = self.row("LayerThumbnail", id)? else { return Ok(None) };
306        let Some(offscreen) = self.offscreen(row.int("ThumbnailOffscreen"))? else { return Ok(None) };
307        if offscreen.data.is_none() {
308            return Ok(None);
309        }
310        offscreen.to_rgba().map(Some)
311    }
312
313    /// The preview CSP saved with the file (`CanvasPreview`, a PNG).
314    pub fn preview(&self) -> Result<Option<RgbaImage>> {
315        let rows = select_all(&self.database, "CanvasPreview")?;
316        let Some(row) = rows.first() else { return Ok(None) };
317        let Some(data) = row.blob("ImageData") else { return Ok(None) };
318        decode_png(data).map(Some)
319    }
320
321    /// Every external chunk id referenced from the database (table, column, id).
322    pub fn external_references(&self) -> Result<Vec<(String, String, String)>> {
323        let schema = self.schema()?;
324        let mut result = Vec::new();
325        for (table, column) in &schema.external_columns {
326            if !database::table_exists(&self.database, table)? || !database::has_column(&self.database, table, column)?
327            {
328                continue;
329            }
330            let sql = format!("SELECT {} FROM {}", database::quote(column), database::quote(table));
331            for row in select(&self.database, &sql, [])? {
332                if let Some(id) = row.values().first().and_then(value_as_text) {
333                    result.push((table.clone(), column.clone(), id));
334                }
335            }
336        }
337        Ok(result)
338    }
339
340    /// The 16-byte file id in the `Head` chunk.
341    pub fn file_id(&self) -> [u8; FILE_ID_LENGTH] {
342        self.file_id.unwrap_or_else(|| {
343            self.container
344                .head
345                .as_ref()
346                .and_then(|head| head.file_id.as_slice().try_into().ok())
347                .unwrap_or([0; FILE_ID_LENGTH])
348        })
349    }
350
351    /// Give the saved file another id. CSP gives every "Save As" copy a new
352    /// id and treats files with the same id as the same document: a copy
353    /// opened while the original is open shows the original's cached
354    /// canvas until something is redrawn. Edited copies should get a new id.
355    pub fn set_file_id(&mut self, id: [u8; FILE_ID_LENGTH]) {
356        self.file_id = Some(id);
357    }
358
359    /// Whether anything was changed since the file was opened.
360    pub fn is_modified(&self) -> bool {
361        self.externals_changed
362            || self
363                .file_id
364                .is_some_and(|id| Some(id.as_slice()) != self.container.head.as_ref().map(|h| h.file_id.as_slice()))
365            || self.database.total_changes() != self.changes_at_open
366    }
367
368    /// The file's bytes. Unchanged files are returned as read.
369    pub fn to_bytes(&self) -> Result<Vec<u8>> {
370        if !self.is_modified() {
371            return Ok(self.container.bytes().to_vec());
372        }
373        let mut writer = ContainerWriter::new(self.file_id());
374        for (id, data) in &self.externals {
375            let payload = match data {
376                External::Original(range) => &self.container.bytes()[range.clone()],
377                External::New(bytes) => bytes.as_slice(),
378            };
379            writer.push_external(id.clone(), payload);
380        }
381        let offsets: HashMap<String, u64> = writer.external_offsets().into_iter().collect();
382        // Offsets are written into a copy so that `self` stays unchanged.
383        let image = database::serialize(&self.database)?;
384        let copy = database::open_bytes(&image)?;
385        update_external_index(&copy, &offsets)?;
386        writer.assemble(&database::serialize(&copy)?)
387    }
388
389    /// Writes [`ClipFile::to_bytes`] to a file.
390    ///
391    /// `path` is replaced if it exists, and nothing checks whether it is
392    /// the file this was read from. Saving over the source is possible (its
393    /// bytes were read into memory), but a failed or interrupted write then
394    /// loses the original; keeping inputs intact is the caller's job. The
395    /// `cspf` tool refuses an output that is one of its inputs.
396    pub fn save(&self, path: impl AsRef<Path>) -> Result<()> {
397        let bytes = self.to_bytes()?;
398        std::fs::write(path, bytes)?;
399        Ok(())
400    }
401}
402
403/// Make `ExternalChunk` list exactly the given ids with their offsets.
404fn update_external_index(database: &Connection, offsets: &HashMap<String, u64>) -> Result<()> {
405    let rows = select_all(database, "ExternalChunk")?;
406    let mut present = std::collections::HashSet::new();
407    for row in &rows {
408        let Some(id) = row.get("ExternalID").and_then(value_as_text) else { continue };
409        let raw = row.get("ExternalID").cloned().unwrap_or(rusqlite::types::Value::Null);
410        match offsets.get(&id) {
411            Some(offset) => {
412                if row.int("Offset") != *offset as i64 {
413                    database.execute(
414                        "UPDATE ExternalChunk SET Offset = ?1 WHERE ExternalID = ?2",
415                        (*offset as i64, &raw),
416                    )?;
417                }
418                present.insert(id);
419            }
420            None => {
421                database.execute("DELETE FROM ExternalChunk WHERE ExternalID = ?1", [&raw])?;
422            }
423        }
424    }
425    let mut missing: Vec<(&String, &u64)> = offsets.iter().filter(|(id, _)| !present.contains(*id)).collect();
426    missing.sort_by_key(|(_, offset)| **offset);
427    // CSP stores ExternalID as TEXT (the column is declared BLOB) and looks
428    // chunks up by text, so a BLOB id is never found: the layer then loads
429    // as "broken".
430    for (id, offset) in missing {
431        database
432            .execute("INSERT INTO ExternalChunk(ExternalID, Offset) VALUES (?1, ?2)", (id.as_str(), *offset as i64))?;
433    }
434    Ok(())
435}
436
437#[cfg(test)]
438mod tests {
439    use super::*;
440
441    #[test]
442    fn new_external_ids_are_indexed_as_text() {
443        let database = Connection::open_in_memory().unwrap();
444        database.execute("CREATE TABLE ExternalChunk(ExternalID BLOB, Offset INTEGER)", []).unwrap();
445        database.execute("INSERT INTO ExternalChunk VALUES ('extrnlidOLD', 10), ('extrnlidGONE', 20)", []).unwrap();
446        let offsets = HashMap::from([("extrnlidOLD".to_owned(), 30), ("extrnlidNEW".to_owned(), 40)]);
447        update_external_index(&database, &offsets).unwrap();
448        let rows: Vec<(String, String, i64)> = database
449            .prepare("SELECT ExternalID, typeof(ExternalID), Offset FROM ExternalChunk ORDER BY Offset")
450            .unwrap()
451            .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
452            .unwrap()
453            .collect::<std::result::Result<_, _>>()
454            .unwrap();
455        assert_eq!(rows, [("extrnlidOLD".into(), "text".into(), 30), ("extrnlidNEW".into(), "text".into(), 40)]);
456    }
457}
csp_format/clip.rs

csp_format/
clip.rs
