csp_format/
container.rs

1//! The CSF chunk container ("CSFCHUNK") that wraps every .clip file.
2//!
3//! All numbers are big endian (spec: `docs/spec/01-container.md`):
4//!
5//! ```text
6//! file header (24 bytes): "CSFCHUNK"  u64 file size  u64 offset of the first chunk (24)
7//! chunk:                  8-byte tag  u64 body length  body
8//!   CHNKHead  body 40 bytes: u64 256, u64 offset of the CHNKSQLi chunk header,
9//!             u64 16, 16-byte file id
10//!   CHNKExta  body: u64 40, 40-byte external id ("extrnlid" + 32 hex digits), payload
11//!   CHNKSQLi  body: SQLite 3 database image
12//!   CHNKFoot  body: empty; the last chunk
13//! ```
14//!
15//! CSP writes Head, every Exta, SQLi, Foot in this order. The table
16//! `ExternalChunk` in the database maps each external id to the file offset
17//! of its CHNKExta header. The parser keeps the original bytes, so a file
18//! that is not changed is written back byte for byte.
19
20use std::collections::HashMap;
21use std::ops::Range;
22use std::path::Path;
23
24use crate::binary::{Reader, Writer};
25use crate::error::{Error, Result, bail};
26
27/// The first 8 bytes of a .clip.
28pub const FILE_MAGIC: &[u8; 8] = b"CSFCHUNK";
29/// Size of the file header.
30pub const FILE_HEADER_SIZE: usize = 24;
31/// Size of a chunk header (tag and body length).
32pub const CHUNK_HEADER_SIZE: usize = 16;
33/// Length of an external id.
34pub const EXTERNAL_ID_LENGTH: usize = 40;
35/// How every external id starts.
36pub const EXTERNAL_ID_PREFIX: &str = "extrnlid";
37/// Size of the CHNKHead body.
38pub const HEAD_BODY_SIZE: usize = 40;
39/// First value of the CHNKHead body.
40pub const HEAD_VERSION: u64 = 256;
41/// Length of the file id in CHNKHead.
42pub const FILE_ID_LENGTH: usize = 16;
43
44/// Tag of the head chunk.
45pub const TAG_HEAD: &[u8; 8] = b"CHNKHead";
46/// Tag of an external chunk.
47pub const TAG_EXTERNAL: &[u8; 8] = b"CHNKExta";
48/// Tag of the database chunk.
49pub const TAG_SQLITE: &[u8; 8] = b"CHNKSQLi";
50/// Tag of the last chunk.
51pub const TAG_FOOTER: &[u8; 8] = b"CHNKFoot";
52
53/// Kind of a chunk, from its 8-byte tag.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55#[non_exhaustive]
56pub enum ChunkKind {
57    /// `CHNKHead`.
58    Head,
59    /// `CHNKExta`.
60    External,
61    /// `CHNKSQLi`.
62    Sqlite,
63    /// `CHNKFoot`.
64    Footer,
65    /// Any other tag.
66    Other([u8; 8]),
67}
68
69impl ChunkKind {
70    /// The kind of a tag.
71    pub fn from_tag(tag: &[u8; 8]) -> Self {
72        match tag {
73            TAG_HEAD => ChunkKind::Head,
74            TAG_EXTERNAL => ChunkKind::External,
75            TAG_SQLITE => ChunkKind::Sqlite,
76            TAG_FOOTER => ChunkKind::Footer,
77            other => ChunkKind::Other(*other),
78        }
79    }
80
81    /// The 8-byte tag.
82    pub fn tag(&self) -> [u8; 8] {
83        match self {
84            ChunkKind::Head => *TAG_HEAD,
85            ChunkKind::External => *TAG_EXTERNAL,
86            ChunkKind::Sqlite => *TAG_SQLITE,
87            ChunkKind::Footer => *TAG_FOOTER,
88            ChunkKind::Other(tag) => *tag,
89        }
90    }
91
92    /// The tag as text.
93    pub fn name(&self) -> String {
94        String::from_utf8_lossy(&self.tag()).into_owned()
95    }
96}
97
98/// One chunk: where its header and body are in the file.
99#[derive(Debug, Clone, PartialEq, Eq)]
100pub struct Chunk {
101    /// Kind, from the tag.
102    pub kind: ChunkKind,
103    /// File offset of the chunk header.
104    pub header_offset: usize,
105    /// File range of the body.
106    pub body: Range<usize>,
107    /// For CHNKExta: the external id and where its payload starts.
108    pub external_id: Option<String>,
109    /// File offset where the payload starts (after the id, for CHNKExta).
110    pub payload_offset: usize,
111}
112
113/// Contents of the CHNKHead chunk.
114#[derive(Debug, Clone, PartialEq, Eq)]
115pub struct Head {
116    /// Always 256 in the corpus (format version, presumably).
117    pub version: u64,
118    /// File offset of the CHNKSQLi chunk header.
119    pub sqlite_offset: u64,
120    /// Length of the file id (always 16).
121    pub file_id_length: u64,
122    /// Random-looking id written when the file is created.
123    pub file_id: Vec<u8>,
124}
125
126/// A parsed .clip container. Holds the original bytes.
127#[derive(Debug, Clone)]
128pub struct Container {
129    data: Vec<u8>,
130    /// File size the header declares.
131    pub declared_size: u64,
132    /// Offset of the first chunk the header declares (24).
133    pub first_chunk: u64,
134    /// The CHNKHead contents, if the file has one.
135    pub head: Option<Head>,
136    /// Every chunk, in file order.
137    pub chunks: Vec<Chunk>,
138    externals: HashMap<String, usize>,
139    /// Bytes after the footer chunk (none in CSP files).
140    pub trailing: usize,
141}
142
143impl Container {
144    /// Reads and parses a file.
145    pub fn read(path: impl AsRef<Path>) -> Result<Self> {
146        let path = path.as_ref();
147        let data =
148            std::fs::read(path).map_err(|error| Error::format(format!("Cannot read {}: {error}", path.display())))?;
149        Self::parse(data)
150    }
151
152    /// Whether `data` starts like a .clip container.
153    pub fn sniff(data: &[u8]) -> bool {
154        data.starts_with(FILE_MAGIC)
155    }
156
157    /// Parses the bytes of a .clip; fails on a truncated or malformed container.
158    pub fn parse(data: Vec<u8>) -> Result<Self> {
159        if data.len() < FILE_HEADER_SIZE || !Self::sniff(&data) {
160            bail!("Not a CLIP STUDIO file (no CSFCHUNK header)");
161        }
162        let mut reader = Reader::at(&data, 8);
163        let declared_size = reader.be_u64()?;
164        let first_chunk = reader.be_u64()?;
165        let mut chunks = Vec::new();
166        let mut externals = HashMap::new();
167        let mut head = None;
168        let mut position = usize::try_from(first_chunk).map_err(|_| Error::format("Bad first chunk offset"))?;
169        let mut ended = false;
170        while position < data.len() {
171            if position + CHUNK_HEADER_SIZE > data.len() {
172                bail!("Truncated chunk header at {position}");
173            }
174            let mut reader = Reader::at(&data, position);
175            let tag: [u8; 8] = reader.bytes(8)?.try_into().expect("8 bytes");
176            if !tag.starts_with(b"CHNK") {
177                bail!("Unexpected chunk tag {:?} at {position}", String::from_utf8_lossy(&tag));
178            }
179            let length = usize::try_from(reader.be_u64()?).map_err(|_| Error::format("Chunk too long"))?;
180            let body_start = position + CHUNK_HEADER_SIZE;
181            let body_end = body_start
182                .checked_add(length)
183                .filter(|end| *end <= data.len())
184                .ok_or_else(|| Error::format(format!("Chunk at {position} overruns the file")))?;
185            let kind = ChunkKind::from_tag(&tag);
186            let mut chunk = Chunk {
187                kind,
188                header_offset: position,
189                body: body_start..body_end,
190                external_id: None,
191                payload_offset: body_start,
192            };
193            match kind {
194                ChunkKind::Head => {
195                    let mut body = Reader::new(&data[body_start..body_end]);
196                    let version = body.be_u64()?;
197                    let sqlite_offset = body.be_u64()?;
198                    let file_id_length = body.be_u64()?;
199                    let file_id = body.bytes(file_id_length.min(body.remaining() as u64) as usize)?.to_vec();
200                    head = Some(Head { version, sqlite_offset, file_id_length, file_id });
201                }
202                ChunkKind::External => {
203                    let mut body = Reader::new(&data[body_start..body_end]);
204                    let id_length = body.be_u64()? as usize;
205                    let id = body.bytes(id_length)?;
206                    let id = String::from_utf8_lossy(id).into_owned();
207                    chunk.payload_offset = body_start + 8 + id_length;
208                    externals.insert(id.clone(), chunks.len());
209                    chunk.external_id = Some(id);
210                }
211                _ => {}
212            }
213            chunks.push(chunk);
214            position = body_end;
215            if kind == ChunkKind::Footer {
216                ended = true;
217                break;
218            }
219        }
220        if !ended {
221            bail!("No CHNKFoot chunk: the file is truncated");
222        }
223        let trailing = data.len() - position;
224        Ok(Container { data, declared_size, first_chunk, head, chunks, externals, trailing })
225    }
226
227    /// The bytes as read.
228    pub fn bytes(&self) -> &[u8] {
229        &self.data
230    }
231
232    /// The bytes as read, by value.
233    pub fn into_bytes(self) -> Vec<u8> {
234        self.data
235    }
236
237    /// The body of a chunk.
238    pub fn body(&self, chunk: &Chunk) -> &[u8] {
239        &self.data[chunk.body.clone()]
240    }
241
242    /// The embedded SQLite database image.
243    pub fn sqlite(&self) -> Result<&[u8]> {
244        self.chunks
245            .iter()
246            .find(|chunk| chunk.kind == ChunkKind::Sqlite)
247            .map(|chunk| self.body(chunk))
248            .ok_or_else(|| Error::format("No CHNKSQLi chunk"))
249    }
250
251    /// The database chunk.
252    pub fn sqlite_chunk(&self) -> Option<&Chunk> {
253        self.chunks.iter().find(|chunk| chunk.kind == ChunkKind::Sqlite)
254    }
255
256    /// External chunks in file order.
257    pub fn external_chunks(&self) -> impl Iterator<Item = &Chunk> {
258        self.chunks.iter().filter(|chunk| chunk.kind == ChunkKind::External)
259    }
260
261    /// The external chunk with this id.
262    pub fn external_chunk(&self, id: &str) -> Option<&Chunk> {
263        self.externals.get(id).map(|index| &self.chunks[*index])
264    }
265
266    /// Payload of an external chunk (after its id). None when the id is
267    /// not stored: CSP keeps ids of offscreens it will build on opening.
268    pub fn external(&self, id: &str) -> Option<&[u8]> {
269        let chunk = self.external_chunk(id)?;
270        Some(&self.data[chunk.payload_offset..chunk.body.end])
271    }
272
273    /// Ids of the external chunks, in file order.
274    pub fn external_ids(&self) -> impl Iterator<Item = &str> {
275        self.external_chunks().filter_map(|chunk| chunk.external_id.as_deref())
276    }
277
278    /// Structural problems that do not stop reading (empty when CSP-like).
279    pub fn findings(&self) -> Vec<String> {
280        let mut findings = Vec::new();
281        if self.declared_size != self.data.len() as u64 {
282            findings.push(format!(
283                "declared size {} differs from the file size {}",
284                self.declared_size,
285                self.data.len()
286            ));
287        }
288        if self.first_chunk != FILE_HEADER_SIZE as u64 {
289            findings.push(format!("first chunk at {} (expected 24)", self.first_chunk));
290        }
291        if self.trailing != 0 {
292            findings.push(format!("{} bytes after CHNKFoot", self.trailing));
293        }
294        match &self.head {
295            None => findings.push("no CHNKHead chunk".into()),
296            Some(head) => {
297                if head.version != HEAD_VERSION {
298                    findings.push(format!("CHNKHead version {}", head.version));
299                }
300                let sqlite = self.sqlite_chunk().map(|chunk| chunk.header_offset as u64);
301                if sqlite != Some(head.sqlite_offset) {
302                    findings.push(format!("CHNKHead points at {} but CHNKSQLi is at {sqlite:?}", head.sqlite_offset));
303                }
304            }
305        }
306        let order: Vec<ChunkKind> = self.chunks.iter().map(|chunk| chunk.kind).collect();
307        let expected_order = order.first() == Some(&ChunkKind::Head)
308            && order.last() == Some(&ChunkKind::Footer)
309            && order.len() >= 3
310            && order[order.len() - 2] == ChunkKind::Sqlite
311            && order[1..order.len() - 2].iter().all(|kind| *kind == ChunkKind::External);
312        if !expected_order {
313            findings.push("chunks are not in the order Head, Exta*, SQLi, Foot".into());
314        }
315        for chunk in self.external_chunks() {
316            let id = chunk.external_id.as_deref().unwrap_or("");
317            if !is_external_id(id) {
318                findings.push(format!("external id {id:?} is not \"extrnlid\" + 32 hex digits"));
319            }
320        }
321        if self.externals.len() != self.external_chunks().count() {
322            findings.push("duplicate external ids".into());
323        }
324        findings
325    }
326
327    /// Rebuild the file from its parsed parts (used to prove the parser
328    /// misses nothing: the result must equal [`Container::bytes`]).
329    pub fn reassemble(&self) -> Vec<u8> {
330        let mut writer = Writer::new();
331        writer.bytes(FILE_MAGIC).be_u64(self.declared_size).be_u64(self.first_chunk);
332        writer.bytes(&self.data[FILE_HEADER_SIZE..self.first_chunk as usize]);
333        for chunk in &self.chunks {
334            writer.bytes(&chunk.kind.tag()).be_u64(chunk.body.len() as u64);
335            match (&chunk.kind, &self.head) {
336                (ChunkKind::Head, Some(head)) if chunk.body.len() == 24 + head.file_id.len() => {
337                    writer
338                        .be_u64(head.version)
339                        .be_u64(head.sqlite_offset)
340                        .be_u64(head.file_id_length)
341                        .bytes(&head.file_id);
342                }
343                (ChunkKind::External, _) => {
344                    let id = chunk.external_id.as_deref().unwrap_or("");
345                    writer.be_u64(id.len() as u64).bytes(id.as_bytes());
346                    writer.bytes(&self.data[chunk.payload_offset..chunk.body.end]);
347                }
348                _ => {
349                    writer.bytes(self.body(chunk));
350                }
351            }
352        }
353        writer.bytes(&self.data[self.data.len() - self.trailing..]);
354        writer.into_inner()
355    }
356}
357
358/// Whether `id` looks like the ids CSP writes ("extrnlid" + 32 hex digits).
359pub fn is_external_id(id: &str) -> bool {
360    id.len() == EXTERNAL_ID_LENGTH
361        && id.starts_with(EXTERNAL_ID_PREFIX)
362        && id[EXTERNAL_ID_PREFIX.len()..].bytes().all(|byte| byte.is_ascii_hexdigit())
363}
364
365/// Builds a new container: Head, the external chunks in the given order,
366/// SQLi, Foot, as CSP writes them.
367#[derive(Debug, Clone)]
368pub struct ContainerWriter<'a> {
369    /// File id written into CHNKHead.
370    pub file_id: [u8; FILE_ID_LENGTH],
371    /// External chunks (id, payload), in file order.
372    pub externals: Vec<(String, &'a [u8])>,
373}
374
375impl<'a> ContainerWriter<'a> {
376    /// A writer with no external chunks.
377    pub fn new(file_id: [u8; FILE_ID_LENGTH]) -> Self {
378        ContainerWriter { file_id, externals: Vec::new() }
379    }
380
381    /// Appends an external chunk.
382    pub fn push_external(&mut self, id: impl Into<String>, payload: &'a [u8]) {
383        self.externals.push((id.into(), payload));
384    }
385
386    /// File offset of each external chunk header (the `ExternalChunk.Offset`
387    /// values). They come before the database, so they do not depend on it.
388    pub fn external_offsets(&self) -> Vec<(String, u64)> {
389        let mut offset = (FILE_HEADER_SIZE + CHUNK_HEADER_SIZE + HEAD_BODY_SIZE) as u64;
390        self.externals
391            .iter()
392            .map(|(id, payload)| {
393                let here = offset;
394                offset += (CHUNK_HEADER_SIZE + 8 + id.len() + payload.len()) as u64;
395                (id.clone(), here)
396            })
397            .collect()
398    }
399
400    /// The finished file, with `sqlite` as the database image.
401    pub fn assemble(&self, sqlite: &[u8]) -> Result<Vec<u8>> {
402        for (id, _) in &self.externals {
403            if !is_external_id(id) {
404                bail!("External id must be \"extrnlid\" + 32 hex digits: {id:?}");
405            }
406        }
407        let payload_size: usize =
408            self.externals.iter().map(|(id, payload)| CHUNK_HEADER_SIZE + 8 + id.len() + payload.len()).sum();
409        let sqlite_offset = FILE_HEADER_SIZE + CHUNK_HEADER_SIZE + HEAD_BODY_SIZE + payload_size;
410        let total = sqlite_offset + CHUNK_HEADER_SIZE + sqlite.len() + CHUNK_HEADER_SIZE;
411        let mut writer = Writer { data: Vec::with_capacity(total) };
412        writer.bytes(FILE_MAGIC).be_u64(total as u64).be_u64(FILE_HEADER_SIZE as u64);
413        writer
414            .bytes(TAG_HEAD)
415            .be_u64(HEAD_BODY_SIZE as u64)
416            .be_u64(HEAD_VERSION)
417            .be_u64(sqlite_offset as u64)
418            .be_u64(FILE_ID_LENGTH as u64)
419            .bytes(&self.file_id);
420        for (id, payload) in &self.externals {
421            writer
422                .bytes(TAG_EXTERNAL)
423                .be_u64((8 + id.len() + payload.len()) as u64)
424                .be_u64(id.len() as u64)
425                .bytes(id.as_bytes())
426                .bytes(payload);
427        }
428        writer.bytes(TAG_SQLITE).be_u64(sqlite.len() as u64).bytes(sqlite);
429        writer.bytes(TAG_FOOTER).be_u64(0);
430        debug_assert_eq!(writer.len(), total);
431        Ok(writer.into_inner())
432    }
433}
434
435#[cfg(test)]
436mod tests {
437    use super::*;
438
439    fn sample() -> Vec<u8> {
440        let first = b"payload one".to_vec();
441        let second = vec![7u8; 300];
442        let mut writer = ContainerWriter::new([9; 16]);
443        writer.push_external(format!("extrnlid{}", "A".repeat(32)), &first);
444        writer.push_external(format!("extrnlid{}", "0".repeat(32)), &second);
445        writer.assemble(b"SQLite format 3\0fake").unwrap()
446    }
447
448    #[test]
449    fn written_containers_parse_and_reassemble() {
450        let data = sample();
451        let container = Container::parse(data.clone()).unwrap();
452        assert!(container.findings().is_empty(), "{:?}", container.findings());
453        assert_eq!(container.reassemble(), data);
454        assert_eq!(container.sqlite().unwrap(), b"SQLite format 3\0fake");
455        let id = format!("extrnlid{}", "A".repeat(32));
456        assert_eq!(container.external(&id).unwrap(), b"payload one");
457        let offsets = ContainerWriter { file_id: [9; 16], externals: vec![(id.clone(), b"payload one".as_slice())] }
458            .external_offsets();
459        assert_eq!(offsets[0].1 as usize, container.external_chunk(&id).unwrap().header_offset);
460    }
461
462    #[test]
463    fn truncated_files_are_rejected() {
464        let data = sample();
465        for cut in [10, 30, data.len() - 20, data.len() - 1] {
466            assert!(Container::parse(data[..cut].to_vec()).is_err(), "cut at {cut}");
467        }
468        assert!(Container::parse(b"SQLite format 3\0".to_vec()).is_err());
469    }
470
471    #[test]
472    fn bad_ids_are_refused_when_writing() {
473        let mut writer = ContainerWriter::new([0; 16]);
474        writer.push_external("short", b"");
475        assert!(writer.assemble(b"").is_err());
476    }
477}
csp_format/container.rs

csp_format/
container.rs
