Skip to main content

rustango_cms/starter/
package.rs

1//! Theme packages: a site's look (and optionally its pages) as one `.zip`
2//! that another site installs.
3//!
4//! A package holds `theme.json` (the [`StarterSeed`](super::StarterSeed)
5//! format) and the site's `templates/`. [`export`] builds the seed from a
6//! live site, [`pack`] / [`unpack`] write and read the archive, and
7//! [`install`] applies it:
8//!
9//! - on an **empty** site, everything: page types, pages, menus, site
10//!   settings and colours, exactly like a starter theme;
11//! - on a site **with content**, only the look: missing page types are
12//!   added, existing ones point at the package's templates, its colours
13//!   become the default, and its error pages fill in any status the site
14//!   has none for. Other pages, menus and settings are left as they are.
15
16use std::collections::{HashMap, HashSet};
17use std::io::{Read as _, Write as _};
18use std::path::{Component, Path, PathBuf};
19
20use rustango::core::Column as _;
21use rustango::sql::{FetcherPool as _, Pool};
22use serde_json::{json, Map, Value};
23
24use super::{create_page_type, create_theme, Outcome, StarterError, MARKER_SCOPE, SITE_SCOPE, UI_APP_LABEL};
25use crate::navigation::{Menu, MenuItem};
26use crate::page::Page;
27use crate::page_builder::model as pb;
28use crate::page_type_model::PageType;
29use crate::theme::Theme;
30
31/// The archive entry holding the seed.
32pub const SEED_ENTRY: &str = "theme.json";
33/// The archive folder holding the templates.
34pub const TEMPLATES_DIR: &str = "templates";
35
36/// Widgets whose stored value is an id of a row on *this* site (an image,
37/// a document, a page, …). Exported content drops them: the ids mean
38/// nothing on another site.
39const REFERENCE_WIDGETS: &[&str] = &[
40    "mediapicker",
41    "documentchooser",
42    "pagechooser",
43    "snippetchooser",
44    "modelchooser",
45    "file",
46];
47/// Id keys of the built-in blocks (image, document, page and snippet
48/// choosers), dropped for the same reason.
49const REFERENCE_BLOCK_KEYS: &[&str] = &["media_id", "document_id", "page_id", "snippet_id"];
50/// Theme columns that identify the row rather than describe the look.
51const THEME_IDENTITY: &[&str] = &["id", "slug", "name", "is_default", "is_admin_default", "created_at", "updated_at"];
52
53/// The site's look, and optionally its pages, as a `theme.json` value.
54///
55/// Page types are the UI-defined ones (those a theme can recreate);
56/// colours are the site's default theme; the error pages come along. With `with_pages`, the page
57/// tree, its content and the menus come too, minus references to images
58/// and other rows that only exist on this site.
59///
60/// # Errors
61/// Database errors.
62pub async fn export(pool: &Pool, name: &str, with_pages: bool) -> Result<Value, StarterError> {
63    let types: Vec<PageType> = PageType::objects().fetch(pool).await?;
64    let mut by_id: HashMap<i64, String> = HashMap::new();
65    let mut page_types = Vec::new();
66    let mut reference_keys: HashMap<String, HashSet<String>> = HashMap::new();
67    for t in &types {
68        let Some(id) = t.id.get().copied() else { continue };
69        by_id.insert(id, t.type_name.clone());
70        if t.app_label != UI_APP_LABEL {
71            continue;
72        }
73        let Some(schema) = pb::published_for(pool, id).await? else { continue };
74        reference_keys.insert(t.type_name.clone(), reference_fields(&schema.document));
75        page_types.push(json!({
76            "type_name": t.type_name,
77            "verbose_name": t.verbose_name,
78            "template": t.default_template,
79            "parents": t.allowed_parents(),
80            "children": t.allowed_children(),
81            "schema": schema.document,
82        }));
83    }
84    let exported: HashSet<&str> = page_types.iter().filter_map(|t| t["type_name"].as_str()).collect();
85
86    let mut seed = Map::new();
87    let site = crate::site_setting::get(pool, SITE_SCOPE).await?.map(|s| s.value_json);
88    // The package is named after the site as visitors see it (its
89    // Branding name), when it has one; `name` otherwise. The name travels
90    // too; the logo and icon are this site's media, so they don't.
91    let branding = crate::branding::get(pool).await;
92    let site_name = branding["site_name"].as_str().map(str::trim).filter(|n| !n.is_empty());
93    seed.insert("name".into(), json!(site_name.unwrap_or(name)));
94    if let Some(n) = site_name {
95        seed.insert("branding".into(), json!({ "site_name": n }));
96    }
97    if let Some(site) = site {
98        seed.insert("site".into(), site);
99    }
100    if let Some(theme) = default_theme(pool).await? {
101        seed.insert("theme".into(), theme_seed(&theme)?);
102    }
103    seed.insert("page_types".into(), Value::Array(page_types.clone()));
104    // The error pages are part of the look: they travel even without the
105    // rest of the content.
106    let home = Page::objects()
107        .where_(Page::depth.eq(1))
108        .order_by(&[("sort_order", false), ("id", false)])
109        .first(pool)
110        .await?;
111    if let Some(home) = home {
112        let error_pages = super::errors::export(pool, &home).await?;
113        if !error_pages.is_empty() {
114            seed.insert("error_pages".into(), Value::Array(error_pages));
115        }
116    }
117
118    if with_pages {
119        let pages: Vec<Page> = Page::objects().fetch(pool).await?;
120        // Only pages whose type travels with the package; a page of a code
121        // type would fail to load on a site without that code.
122        let travels = |p: &Page| by_id.get(&p.page_type_id).is_some_and(|t| exported.contains(t.as_str()));
123        let mut keys: HashMap<i64, String> = HashMap::new();
124        for p in &pages {
125            if let Some(id) = p.id.get().copied() {
126                keys.insert(id, page_key(p));
127            }
128        }
129        let roots: Vec<&Page> = pages.iter().filter(|p| p.parent_id.is_none() && travels(p)).collect();
130        let mut tree = Vec::new();
131        for root in roots.into_iter().take(1) {
132            tree.push(page_seed(pool, root, &pages, &by_id, &keys, &reference_keys, &travels).await?);
133        }
134        seed.insert("pages".into(), Value::Array(tree));
135        seed.insert("menus".into(), Value::Array(menus(pool, &keys).await?));
136    }
137    Ok(Value::Object(seed))
138}
139
140/// The most entries and the most unpacked bytes a package may hold. A theme
141/// is a few dozen small text files; anything far beyond that is refused
142/// before it is unpacked, so a hostile archive cannot fill the disk.
143pub const MAX_ENTRIES: usize = 500;
144pub const MAX_UNPACKED_BYTES: u64 = 8 * 1024 * 1024;
145
146/// A package read into memory: its `theme.json` and its templates, each as
147/// (path under `templates/`, body).
148#[derive(Debug)]
149pub struct Package {
150    pub seed: String,
151    pub templates: Vec<(String, String)>,
152}
153
154/// Build a package in memory.
155///
156/// # Errors
157/// Serialisation or archive errors.
158pub fn pack_bytes(seed: &Value, templates: &[(String, String)]) -> Result<Vec<u8>, StarterError> {
159    let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
160    let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
161    zip.start_file(SEED_ENTRY, opts).map_err(zip_err)?;
162    zip.write_all(serde_json::to_string_pretty(seed)?.as_bytes()).map_err(io)?;
163    for (name, body) in templates {
164        let rel = safe_relative(name)
165            .ok_or_else(|| StarterError::Invalid(format!("template name `{name}` is not a relative path")))?;
166        let rel = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
167        zip.start_file(format!("{TEMPLATES_DIR}/{rel}"), opts).map_err(zip_err)?;
168        zip.write_all(body.as_bytes()).map_err(io)?;
169    }
170    Ok(zip.finish().map_err(zip_err)?.into_inner())
171}
172
173/// Read a package from memory. Refuses an archive with too many entries
174/// or too many unpacked bytes, an entry outside `templates/`, a path that
175/// would escape its folder, a template that is not UTF-8 text, and an
176/// archive without `theme.json`.
177///
178/// # Errors
179/// [`StarterError::Invalid`] naming the problem.
180pub fn read_bytes(bytes: &[u8]) -> Result<Package, StarterError> {
181    let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(zip_err)?;
182    if zip.len() > MAX_ENTRIES {
183        return Err(StarterError::Invalid(format!(
184            "package has {} files; a theme has at most {MAX_ENTRIES}",
185            zip.len()
186        )));
187    }
188    let mut seed = None;
189    let mut templates = Vec::new();
190    let mut total: u64 = 0;
191    for i in 0..zip.len() {
192        let entry = zip.by_index(i).map_err(zip_err)?;
193        if entry.is_dir() {
194            continue;
195        }
196        let name = entry.name().map_err(zip_err)?.into_owned();
197        // Read at most what is left of the budget, plus one byte to notice
198        // an entry that lies about its size.
199        let budget = MAX_UNPACKED_BYTES.saturating_sub(total);
200        let mut body = Vec::new();
201        entry.take(budget + 1).read_to_end(&mut body).map_err(io)?;
202        total += body.len() as u64;
203        if total > MAX_UNPACKED_BYTES {
204            return Err(StarterError::Invalid(format!(
205                "package unpacks to more than {} MB",
206                MAX_UNPACKED_BYTES / (1024 * 1024)
207            )));
208        }
209        let text = String::from_utf8(body)
210            .map_err(|_| StarterError::Invalid(format!("package entry `{name}` is not text")))?;
211        if name == SEED_ENTRY {
212            seed = Some(text);
213            continue;
214        }
215        let rel = name
216            .strip_prefix(&format!("{TEMPLATES_DIR}/"))
217            .and_then(safe_relative)
218            .ok_or_else(|| StarterError::Invalid(format!("package entry `{name}` is not a template")))?;
219        let rel = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
220        templates.push((rel, text));
221    }
222    let seed = seed.ok_or_else(|| StarterError::Invalid(format!("package has no {SEED_ENTRY}")))?;
223    Ok(Package { seed, templates })
224}
225
226/// Write a package file: `theme.json` plus every file under `templates_dir`.
227///
228/// # Errors
229/// I/O errors reading the templates or writing the archive.
230pub fn pack(seed: &Value, templates_dir: &Path, out: &Path) -> Result<usize, StarterError> {
231    let mut files = Vec::new();
232    collect(templates_dir, &mut files).map_err(io)?;
233    files.sort();
234    let mut templates = Vec::with_capacity(files.len());
235    for path in &files {
236        let rel = path.strip_prefix(templates_dir).unwrap_or(path);
237        let name = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
238        let body = std::fs::read_to_string(path)
239            .map_err(|e| StarterError::Invalid(format!("{}: {e}", path.display())))?;
240        templates.push((name, body));
241    }
242    std::fs::write(out, pack_bytes(seed, &templates)?).map_err(io)?;
243    Ok(templates.len())
244}
245
246/// Read a package file, writing its templates under `templates_dest`
247/// (created if missing); returns the `theme.json` and how many templates
248/// were written. Same refusals as [`read_bytes`].
249///
250/// # Errors
251/// An invalid package or I/O errors.
252pub fn unpack(package: &Path, templates_dest: &Path) -> Result<(String, usize), StarterError> {
253    let pkg = read_bytes(&std::fs::read(package).map_err(io)?)?;
254    for (name, body) in &pkg.templates {
255        let dest = templates_dest.join(name);
256        if let Some(dir) = dest.parent() {
257            std::fs::create_dir_all(dir).map_err(io)?;
258        }
259        std::fs::write(&dest, body).map_err(io)?;
260    }
261    Ok((pkg.seed, pkg.templates.len()))
262}
263
264/// Read only the `theme.json` of a package file.
265///
266/// # Errors
267/// An invalid package or I/O errors.
268pub fn read_seed(package: &Path) -> Result<String, StarterError> {
269    Ok(read_bytes(&std::fs::read(package).map_err(io)?)?.seed)
270}
271
272/// Apply a package's `theme.json` to a site: everything on an empty site,
273/// only the look on a site with content (see the module docs).
274///
275/// # Errors
276/// An invalid seed or database errors.
277pub async fn install(pool: &Pool, seed_json: &str) -> Result<Outcome, StarterError> {
278    let site_is_empty = crate::site_setting::get(pool, MARKER_SCOPE).await?.is_none()
279        && Page::objects().first(pool).await?.is_none();
280    if site_is_empty {
281        return super::apply(pool, seed_json).await;
282    }
283    let seed: super::StarterSeed = serde_json::from_str(seed_json)?;
284    super::check_types(&seed, &pb::component_map(pool).await?)?;
285
286    let mut added = 0;
287    let mut retemplated = 0;
288    for pt in &seed.page_types {
289        let existing = PageType::objects()
290            .where_(PageType::type_name.eq(pt.type_name.clone()))
291            .first(pool)
292            .await?;
293        match existing {
294            None => {
295                create_page_type(pool, pt).await?;
296                added += 1;
297            }
298            // A type made in the admin (or by a theme): its fields and
299            // content stay; it renders with the package's template.
300            Some(mut row) if row.app_label == UI_APP_LABEL => {
301                if row.default_template != pt.template {
302                    row.default_template = pt.template.clone();
303                    row.save(pool).await?;
304                    retemplated += 1;
305                }
306            }
307            // A type defined in code keeps the template its code names.
308            Some(_) => {}
309        }
310    }
311    let roots = Page::objects()
312        .where_(Page::depth.eq(1))
313        .order_by(&[("sort_order", false), ("id", false)])
314        .fetch(pool)
315        .await?;
316    if let Some(theme) = &seed.theme {
317        let theme_id = create_theme(pool, theme).await?;
318        for mut root in roots.iter().cloned() {
319            root.theme_id = Some(theme_id);
320            root.save(pool).await?;
321        }
322    }
323    let error_pages_added = match roots.first() {
324        Some(home) => super::errors::add_missing(pool, home, &seed.error_pages).await?,
325        None => 0,
326    };
327    crate::site_setting::upsert(
328        pool,
329        MARKER_SCOPE,
330        json!({ "theme": seed.name, "applied_at": chrono::Utc::now().to_rfc3339() }),
331    )
332    .await?;
333    Ok(Outcome::Switched {
334        page_types_added: added,
335        page_types_retemplated: retemplated,
336        error_pages_added,
337    })
338}
339
340async fn default_theme(pool: &Pool) -> Result<Option<Theme>, StarterError> {
341    Ok(Theme::objects().where_(Theme::is_default.eq(true)).first(pool).await?)
342}
343
344fn theme_seed(theme: &Theme) -> Result<Value, StarterError> {
345    let Value::Object(mut tokens) = serde_json::to_value(theme)? else {
346        return Err(StarterError::Invalid("theme did not serialize to an object".into()));
347    };
348    for key in THEME_IDENTITY {
349        tokens.remove(*key);
350    }
351    Ok(json!({
352        "base": "paper",
353        "slug": theme.slug,
354        "name": theme.name,
355        "tokens": tokens,
356    }))
357}
358
359/// A stable, readable key for a page: its URL path, `home` for the root.
360fn page_key(p: &Page) -> String {
361    let trimmed = p.url_path.trim_matches('/');
362    if trimmed.is_empty() {
363        "home".to_owned()
364    } else {
365        trimmed.replace('/', "-")
366    }
367}
368
369fn page_seed<'a>(
370    pool: &'a Pool,
371    page: &'a Page,
372    all: &'a [Page],
373    type_names: &'a HashMap<i64, String>,
374    keys: &'a HashMap<i64, String>,
375    reference_keys: &'a HashMap<String, HashSet<String>>,
376    travels: &'a (dyn Fn(&Page) -> bool + Sync),
377) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Value, StarterError>> + Send + 'a>> {
378    Box::pin(async move {
379        let id = page.id.get().copied().unwrap_or_default();
380        let type_name = type_names.get(&page.page_type_id).cloned().unwrap_or_default();
381        let mut seed = json!({
382            "key": keys.get(&id).cloned().unwrap_or_default(),
383            "type": type_name,
384            "title": page.title,
385            "slug": page.slug,
386        });
387        if !page.seo_description.is_empty() {
388            seed["seo_description"] = json!(page.seo_description);
389        }
390        if page.show_in_menus {
391            seed["in_menu"] = json!(true);
392        }
393        if let Some(row) = pb::data_for_page(pool, id).await? {
394            let mut data = row.data;
395            strip_references(&mut data, reference_keys.get(&type_name));
396            seed["data"] = data;
397        }
398        let mut children: Vec<&Page> = all.iter().filter(|c| c.parent_id == Some(id) && travels(c)).collect();
399        children.sort_by_key(|c| (c.sort_order, c.path.clone()));
400        let mut out = Vec::new();
401        for child in children {
402            out.push(page_seed(pool, child, all, type_names, keys, reference_keys, travels).await?);
403        }
404        if !out.is_empty() {
405            seed["children"] = Value::Array(out);
406        }
407        Ok(seed)
408    })
409}
410
411async fn menus(pool: &Pool, keys: &HashMap<i64, String>) -> Result<Vec<Value>, StarterError> {
412    let mut out = Vec::new();
413    for menu in Menu::objects().fetch(pool).await? {
414        let Some(menu_id) = menu.id.get().copied() else { continue };
415        let items: Vec<MenuItem> = MenuItem::objects()
416            .where_(MenuItem::menu_id.eq(menu_id))
417            .fetch(pool)
418            .await?;
419        out.push(json!({
420            "slug": menu.slug,
421            "name": menu.name,
422            "items": menu_items(&items, None, keys),
423        }));
424    }
425    Ok(out)
426}
427
428fn menu_items(items: &[MenuItem], parent: Option<i64>, keys: &HashMap<i64, String>) -> Vec<Value> {
429    let mut level: Vec<&MenuItem> = items.iter().filter(|i| i.parent_id == parent).collect();
430    level.sort_by_key(|i| i.sort_order);
431    level
432        .into_iter()
433        .filter_map(|item| {
434            let mut v = json!({ "label": item.label });
435            match (item.page_id.and_then(|id| keys.get(&id)), &item.external_url) {
436                (Some(key), _) => v["page"] = json!(key),
437                // A link to a page that does not travel is dropped.
438                (None, _) if item.page_id.is_some() => return None,
439                (None, Some(url)) => v["url"] = json!(url),
440                (None, None) => {}
441            }
442            let children = menu_items(items, item.id.get().copied(), keys);
443            if v.get("page").is_none() && v.get("url").is_none() && item.label.is_empty() {
444                return None;
445            }
446            if !children.is_empty() {
447                v["children"] = Value::Array(children);
448            }
449            Some(v)
450        })
451        .collect()
452}
453
454/// The keys of every field in a schema whose value is an id of a row on
455/// this site, at any depth (rows, groups, repeaters, flexible zones).
456fn reference_fields(schema: &Value) -> HashSet<String> {
457    let mut keys = HashSet::new();
458    walk_nodes(schema, &mut keys);
459    keys
460}
461
462fn walk_nodes(v: &Value, keys: &mut HashSet<String>) {
463    match v {
464        Value::Object(map) => {
465            if map.get("kind").and_then(Value::as_str) == Some("field")
466                && map
467                    .get("widget")
468                    .and_then(Value::as_str)
469                    .is_some_and(|w| REFERENCE_WIDGETS.contains(&w))
470            {
471                if let Some(k) = map.get("key").and_then(Value::as_str) {
472                    keys.insert(k.to_owned());
473                }
474            }
475            for child in map.values() {
476                walk_nodes(child, keys);
477            }
478        }
479        Value::Array(items) => items.iter().for_each(|i| walk_nodes(i, keys)),
480        _ => {}
481    }
482}
483
484fn strip_references(data: &mut Value, keys: Option<&HashSet<String>>) {
485    match data {
486        Value::Object(map) => {
487            for (k, v) in map.iter_mut() {
488                if REFERENCE_BLOCK_KEYS.contains(&k.as_str()) || keys.is_some_and(|ks| ks.contains(k)) {
489                    *v = Value::Null;
490                } else {
491                    strip_references(v, keys);
492                }
493            }
494        }
495        Value::Array(items) => items.iter_mut().for_each(|i| strip_references(i, keys)),
496        _ => {}
497    }
498}
499
500/// `rel` as a path that stays inside its destination, or `None`.
501fn safe_relative(rel: &str) -> Option<PathBuf> {
502    let path = Path::new(rel);
503    let ok = !rel.is_empty() && path.components().all(|c| matches!(c, Component::Normal(_)));
504    ok.then(|| path.to_path_buf())
505}
506
507fn collect(dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
508    for entry in std::fs::read_dir(dir)? {
509        let path = entry?.path();
510        if path.is_dir() {
511            collect(&path, out)?;
512        } else {
513            out.push(path);
514        }
515    }
516    Ok(())
517}
518
519fn io(e: std::io::Error) -> StarterError {
520    StarterError::Invalid(e.to_string())
521}
522
523fn zip_err(e: zip::result::ZipError) -> StarterError {
524    StarterError::Invalid(e.to_string())
525}
526
527#[cfg(test)]
528mod tests {
529    use super::*;
530
531    #[test]
532    fn unsafe_package_paths_are_refused() {
533        assert!(safe_relative("post.html").is_some());
534        assert!(safe_relative("blocks/cta.html").is_some());
535        for bad in ["", "../etc/passwd", "/etc/passwd", "blocks/../../x", "./x"] {
536            assert!(safe_relative(bad).is_none(), "{bad}");
537        }
538    }
539
540    #[test]
541    fn exported_content_drops_ids_of_this_site() {
542        let schema = json!({ "nodes": [
543            { "kind": "field", "key": "cover", "widget": "mediapicker" },
544            { "kind": "field", "key": "title", "widget": "text" },
545            { "kind": "flex", "key": "body", "allowed": ["image", "gallery"], "groups": [
546                { "key": "gallery", "children": [ { "kind": "field", "key": "image_1", "widget": "mediapicker" } ] } ] }
547        ]});
548        let keys = reference_fields(&schema);
549        let mut data = json!({
550            "cover": 12, "title": "Hi",
551            "body": [
552                { "type": "image", "value": { "media_id": 3, "caption": "c" } },
553                { "type": "gallery", "value": { "image_1": 4 } }
554            ]
555        });
556        strip_references(&mut data, Some(&keys));
557        assert_eq!(data["cover"], Value::Null);
558        assert_eq!(data["title"], "Hi");
559        assert_eq!(data["body"][0]["value"]["media_id"], Value::Null);
560        assert_eq!(data["body"][0]["value"]["caption"], "c");
561        assert_eq!(data["body"][1]["value"]["image_1"], Value::Null);
562    }
563}