1use std::collections::{HashMap, HashSet};
17use std::io::{Read as _, Write as _};
18use std::path::{Component, Path, PathBuf};
19
20use rustango::core::Column as _;
21use rustango::sql::{FetcherPool as _, Pool};
22use serde_json::{json, Map, Value};
23
24use super::{create_page_type, create_theme, Outcome, StarterError, MARKER_SCOPE, SITE_SCOPE, UI_APP_LABEL};
25use crate::navigation::{Menu, MenuItem};
26use crate::page::Page;
27use crate::page_builder::model as pb;
28use crate::page_type_model::PageType;
29use crate::theme::Theme;
30
31pub const SEED_ENTRY: &str = "theme.json";
33pub const TEMPLATES_DIR: &str = "templates";
35
36const REFERENCE_WIDGETS: &[&str] = &[
40 "mediapicker",
41 "documentchooser",
42 "pagechooser",
43 "snippetchooser",
44 "modelchooser",
45 "file",
46];
47const REFERENCE_BLOCK_KEYS: &[&str] = &["media_id", "document_id", "page_id", "snippet_id"];
50const THEME_IDENTITY: &[&str] = &["id", "slug", "name", "is_default", "is_admin_default", "created_at", "updated_at"];
52
53pub async fn export(pool: &Pool, name: &str, with_pages: bool) -> Result<Value, StarterError> {
63 let types: Vec<PageType> = PageType::objects().fetch(pool).await?;
64 let mut by_id: HashMap<i64, String> = HashMap::new();
65 let mut page_types = Vec::new();
66 let mut reference_keys: HashMap<String, HashSet<String>> = HashMap::new();
67 for t in &types {
68 let Some(id) = t.id.get().copied() else { continue };
69 by_id.insert(id, t.type_name.clone());
70 if t.app_label != UI_APP_LABEL {
71 continue;
72 }
73 let Some(schema) = pb::published_for(pool, id).await? else { continue };
74 reference_keys.insert(t.type_name.clone(), reference_fields(&schema.document));
75 page_types.push(json!({
76 "type_name": t.type_name,
77 "verbose_name": t.verbose_name,
78 "template": t.default_template,
79 "parents": t.allowed_parents(),
80 "children": t.allowed_children(),
81 "schema": schema.document,
82 }));
83 }
84 let exported: HashSet<&str> = page_types.iter().filter_map(|t| t["type_name"].as_str()).collect();
85
86 let mut seed = Map::new();
87 let site = crate::site_setting::get(pool, SITE_SCOPE).await?.map(|s| s.value_json);
88 let branding = crate::branding::get(pool).await;
92 let site_name = branding["site_name"].as_str().map(str::trim).filter(|n| !n.is_empty());
93 seed.insert("name".into(), json!(site_name.unwrap_or(name)));
94 if let Some(n) = site_name {
95 seed.insert("branding".into(), json!({ "site_name": n }));
96 }
97 if let Some(site) = site {
98 seed.insert("site".into(), site);
99 }
100 if let Some(theme) = default_theme(pool).await? {
101 seed.insert("theme".into(), theme_seed(&theme)?);
102 }
103 seed.insert("page_types".into(), Value::Array(page_types.clone()));
104 let home = Page::objects()
107 .where_(Page::depth.eq(1))
108 .order_by(&[("sort_order", false), ("id", false)])
109 .first(pool)
110 .await?;
111 if let Some(home) = home {
112 let error_pages = super::errors::export(pool, &home).await?;
113 if !error_pages.is_empty() {
114 seed.insert("error_pages".into(), Value::Array(error_pages));
115 }
116 }
117
118 if with_pages {
119 let pages: Vec<Page> = Page::objects().fetch(pool).await?;
120 let travels = |p: &Page| by_id.get(&p.page_type_id).is_some_and(|t| exported.contains(t.as_str()));
123 let mut keys: HashMap<i64, String> = HashMap::new();
124 for p in &pages {
125 if let Some(id) = p.id.get().copied() {
126 keys.insert(id, page_key(p));
127 }
128 }
129 let roots: Vec<&Page> = pages.iter().filter(|p| p.parent_id.is_none() && travels(p)).collect();
130 let mut tree = Vec::new();
131 for root in roots.into_iter().take(1) {
132 tree.push(page_seed(pool, root, &pages, &by_id, &keys, &reference_keys, &travels).await?);
133 }
134 seed.insert("pages".into(), Value::Array(tree));
135 seed.insert("menus".into(), Value::Array(menus(pool, &keys).await?));
136 }
137 Ok(Value::Object(seed))
138}
139
140pub const MAX_ENTRIES: usize = 500;
144pub const MAX_UNPACKED_BYTES: u64 = 8 * 1024 * 1024;
145
146#[derive(Debug)]
149pub struct Package {
150 pub seed: String,
151 pub templates: Vec<(String, String)>,
152}
153
154pub fn pack_bytes(seed: &Value, templates: &[(String, String)]) -> Result<Vec<u8>, StarterError> {
159 let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
160 let opts = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
161 zip.start_file(SEED_ENTRY, opts).map_err(zip_err)?;
162 zip.write_all(serde_json::to_string_pretty(seed)?.as_bytes()).map_err(io)?;
163 for (name, body) in templates {
164 let rel = safe_relative(name)
165 .ok_or_else(|| StarterError::Invalid(format!("template name `{name}` is not a relative path")))?;
166 let rel = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
167 zip.start_file(format!("{TEMPLATES_DIR}/{rel}"), opts).map_err(zip_err)?;
168 zip.write_all(body.as_bytes()).map_err(io)?;
169 }
170 Ok(zip.finish().map_err(zip_err)?.into_inner())
171}
172
173pub fn read_bytes(bytes: &[u8]) -> Result<Package, StarterError> {
181 let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(zip_err)?;
182 if zip.len() > MAX_ENTRIES {
183 return Err(StarterError::Invalid(format!(
184 "package has {} files; a theme has at most {MAX_ENTRIES}",
185 zip.len()
186 )));
187 }
188 let mut seed = None;
189 let mut templates = Vec::new();
190 let mut total: u64 = 0;
191 for i in 0..zip.len() {
192 let entry = zip.by_index(i).map_err(zip_err)?;
193 if entry.is_dir() {
194 continue;
195 }
196 let name = entry.name().map_err(zip_err)?.into_owned();
197 let budget = MAX_UNPACKED_BYTES.saturating_sub(total);
200 let mut body = Vec::new();
201 entry.take(budget + 1).read_to_end(&mut body).map_err(io)?;
202 total += body.len() as u64;
203 if total > MAX_UNPACKED_BYTES {
204 return Err(StarterError::Invalid(format!(
205 "package unpacks to more than {} MB",
206 MAX_UNPACKED_BYTES / (1024 * 1024)
207 )));
208 }
209 let text = String::from_utf8(body)
210 .map_err(|_| StarterError::Invalid(format!("package entry `{name}` is not text")))?;
211 if name == SEED_ENTRY {
212 seed = Some(text);
213 continue;
214 }
215 let rel = name
216 .strip_prefix(&format!("{TEMPLATES_DIR}/"))
217 .and_then(safe_relative)
218 .ok_or_else(|| StarterError::Invalid(format!("package entry `{name}` is not a template")))?;
219 let rel = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
220 templates.push((rel, text));
221 }
222 let seed = seed.ok_or_else(|| StarterError::Invalid(format!("package has no {SEED_ENTRY}")))?;
223 Ok(Package { seed, templates })
224}
225
226pub fn pack(seed: &Value, templates_dir: &Path, out: &Path) -> Result<usize, StarterError> {
231 let mut files = Vec::new();
232 collect(templates_dir, &mut files).map_err(io)?;
233 files.sort();
234 let mut templates = Vec::with_capacity(files.len());
235 for path in &files {
236 let rel = path.strip_prefix(templates_dir).unwrap_or(path);
237 let name = rel.components().map(|c| c.as_os_str().to_string_lossy()).collect::<Vec<_>>().join("/");
238 let body = std::fs::read_to_string(path)
239 .map_err(|e| StarterError::Invalid(format!("{}: {e}", path.display())))?;
240 templates.push((name, body));
241 }
242 std::fs::write(out, pack_bytes(seed, &templates)?).map_err(io)?;
243 Ok(templates.len())
244}
245
246pub fn unpack(package: &Path, templates_dest: &Path) -> Result<(String, usize), StarterError> {
253 let pkg = read_bytes(&std::fs::read(package).map_err(io)?)?;
254 for (name, body) in &pkg.templates {
255 let dest = templates_dest.join(name);
256 if let Some(dir) = dest.parent() {
257 std::fs::create_dir_all(dir).map_err(io)?;
258 }
259 std::fs::write(&dest, body).map_err(io)?;
260 }
261 Ok((pkg.seed, pkg.templates.len()))
262}
263
264pub fn read_seed(package: &Path) -> Result<String, StarterError> {
269 Ok(read_bytes(&std::fs::read(package).map_err(io)?)?.seed)
270}
271
272pub async fn install(pool: &Pool, seed_json: &str) -> Result<Outcome, StarterError> {
278 let site_is_empty = crate::site_setting::get(pool, MARKER_SCOPE).await?.is_none()
279 && Page::objects().first(pool).await?.is_none();
280 if site_is_empty {
281 return super::apply(pool, seed_json).await;
282 }
283 let seed: super::StarterSeed = serde_json::from_str(seed_json)?;
284 super::check_types(&seed, &pb::component_map(pool).await?)?;
285
286 let mut added = 0;
287 let mut retemplated = 0;
288 for pt in &seed.page_types {
289 let existing = PageType::objects()
290 .where_(PageType::type_name.eq(pt.type_name.clone()))
291 .first(pool)
292 .await?;
293 match existing {
294 None => {
295 create_page_type(pool, pt).await?;
296 added += 1;
297 }
298 Some(mut row) if row.app_label == UI_APP_LABEL => {
301 if row.default_template != pt.template {
302 row.default_template = pt.template.clone();
303 row.save(pool).await?;
304 retemplated += 1;
305 }
306 }
307 Some(_) => {}
309 }
310 }
311 let roots = Page::objects()
312 .where_(Page::depth.eq(1))
313 .order_by(&[("sort_order", false), ("id", false)])
314 .fetch(pool)
315 .await?;
316 if let Some(theme) = &seed.theme {
317 let theme_id = create_theme(pool, theme).await?;
318 for mut root in roots.iter().cloned() {
319 root.theme_id = Some(theme_id);
320 root.save(pool).await?;
321 }
322 }
323 let error_pages_added = match roots.first() {
324 Some(home) => super::errors::add_missing(pool, home, &seed.error_pages).await?,
325 None => 0,
326 };
327 crate::site_setting::upsert(
328 pool,
329 MARKER_SCOPE,
330 json!({ "theme": seed.name, "applied_at": chrono::Utc::now().to_rfc3339() }),
331 )
332 .await?;
333 Ok(Outcome::Switched {
334 page_types_added: added,
335 page_types_retemplated: retemplated,
336 error_pages_added,
337 })
338}
339
340async fn default_theme(pool: &Pool) -> Result<Option<Theme>, StarterError> {
341 Ok(Theme::objects().where_(Theme::is_default.eq(true)).first(pool).await?)
342}
343
344fn theme_seed(theme: &Theme) -> Result<Value, StarterError> {
345 let Value::Object(mut tokens) = serde_json::to_value(theme)? else {
346 return Err(StarterError::Invalid("theme did not serialize to an object".into()));
347 };
348 for key in THEME_IDENTITY {
349 tokens.remove(*key);
350 }
351 Ok(json!({
352 "base": "paper",
353 "slug": theme.slug,
354 "name": theme.name,
355 "tokens": tokens,
356 }))
357}
358
359fn page_key(p: &Page) -> String {
361 let trimmed = p.url_path.trim_matches('/');
362 if trimmed.is_empty() {
363 "home".to_owned()
364 } else {
365 trimmed.replace('/', "-")
366 }
367}
368
369fn page_seed<'a>(
370 pool: &'a Pool,
371 page: &'a Page,
372 all: &'a [Page],
373 type_names: &'a HashMap<i64, String>,
374 keys: &'a HashMap<i64, String>,
375 reference_keys: &'a HashMap<String, HashSet<String>>,
376 travels: &'a (dyn Fn(&Page) -> bool + Sync),
377) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Value, StarterError>> + Send + 'a>> {
378 Box::pin(async move {
379 let id = page.id.get().copied().unwrap_or_default();
380 let type_name = type_names.get(&page.page_type_id).cloned().unwrap_or_default();
381 let mut seed = json!({
382 "key": keys.get(&id).cloned().unwrap_or_default(),
383 "type": type_name,
384 "title": page.title,
385 "slug": page.slug,
386 });
387 if !page.seo_description.is_empty() {
388 seed["seo_description"] = json!(page.seo_description);
389 }
390 if page.show_in_menus {
391 seed["in_menu"] = json!(true);
392 }
393 if let Some(row) = pb::data_for_page(pool, id).await? {
394 let mut data = row.data;
395 strip_references(&mut data, reference_keys.get(&type_name));
396 seed["data"] = data;
397 }
398 let mut children: Vec<&Page> = all.iter().filter(|c| c.parent_id == Some(id) && travels(c)).collect();
399 children.sort_by_key(|c| (c.sort_order, c.path.clone()));
400 let mut out = Vec::new();
401 for child in children {
402 out.push(page_seed(pool, child, all, type_names, keys, reference_keys, travels).await?);
403 }
404 if !out.is_empty() {
405 seed["children"] = Value::Array(out);
406 }
407 Ok(seed)
408 })
409}
410
411async fn menus(pool: &Pool, keys: &HashMap<i64, String>) -> Result<Vec<Value>, StarterError> {
412 let mut out = Vec::new();
413 for menu in Menu::objects().fetch(pool).await? {
414 let Some(menu_id) = menu.id.get().copied() else { continue };
415 let items: Vec<MenuItem> = MenuItem::objects()
416 .where_(MenuItem::menu_id.eq(menu_id))
417 .fetch(pool)
418 .await?;
419 out.push(json!({
420 "slug": menu.slug,
421 "name": menu.name,
422 "items": menu_items(&items, None, keys),
423 }));
424 }
425 Ok(out)
426}
427
428fn menu_items(items: &[MenuItem], parent: Option<i64>, keys: &HashMap<i64, String>) -> Vec<Value> {
429 let mut level: Vec<&MenuItem> = items.iter().filter(|i| i.parent_id == parent).collect();
430 level.sort_by_key(|i| i.sort_order);
431 level
432 .into_iter()
433 .filter_map(|item| {
434 let mut v = json!({ "label": item.label });
435 match (item.page_id.and_then(|id| keys.get(&id)), &item.external_url) {
436 (Some(key), _) => v["page"] = json!(key),
437 (None, _) if item.page_id.is_some() => return None,
439 (None, Some(url)) => v["url"] = json!(url),
440 (None, None) => {}
441 }
442 let children = menu_items(items, item.id.get().copied(), keys);
443 if v.get("page").is_none() && v.get("url").is_none() && item.label.is_empty() {
444 return None;
445 }
446 if !children.is_empty() {
447 v["children"] = Value::Array(children);
448 }
449 Some(v)
450 })
451 .collect()
452}
453
454fn reference_fields(schema: &Value) -> HashSet<String> {
457 let mut keys = HashSet::new();
458 walk_nodes(schema, &mut keys);
459 keys
460}
461
462fn walk_nodes(v: &Value, keys: &mut HashSet<String>) {
463 match v {
464 Value::Object(map) => {
465 if map.get("kind").and_then(Value::as_str) == Some("field")
466 && map
467 .get("widget")
468 .and_then(Value::as_str)
469 .is_some_and(|w| REFERENCE_WIDGETS.contains(&w))
470 {
471 if let Some(k) = map.get("key").and_then(Value::as_str) {
472 keys.insert(k.to_owned());
473 }
474 }
475 for child in map.values() {
476 walk_nodes(child, keys);
477 }
478 }
479 Value::Array(items) => items.iter().for_each(|i| walk_nodes(i, keys)),
480 _ => {}
481 }
482}
483
484fn strip_references(data: &mut Value, keys: Option<&HashSet<String>>) {
485 match data {
486 Value::Object(map) => {
487 for (k, v) in map.iter_mut() {
488 if REFERENCE_BLOCK_KEYS.contains(&k.as_str()) || keys.is_some_and(|ks| ks.contains(k)) {
489 *v = Value::Null;
490 } else {
491 strip_references(v, keys);
492 }
493 }
494 }
495 Value::Array(items) => items.iter_mut().for_each(|i| strip_references(i, keys)),
496 _ => {}
497 }
498}
499
500fn safe_relative(rel: &str) -> Option<PathBuf> {
502 let path = Path::new(rel);
503 let ok = !rel.is_empty() && path.components().all(|c| matches!(c, Component::Normal(_)));
504 ok.then(|| path.to_path_buf())
505}
506
507fn collect(dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
508 for entry in std::fs::read_dir(dir)? {
509 let path = entry?.path();
510 if path.is_dir() {
511 collect(&path, out)?;
512 } else {
513 out.push(path);
514 }
515 }
516 Ok(())
517}
518
519fn io(e: std::io::Error) -> StarterError {
520 StarterError::Invalid(e.to_string())
521}
522
523fn zip_err(e: zip::result::ZipError) -> StarterError {
524 StarterError::Invalid(e.to_string())
525}
526
527#[cfg(test)]
528mod tests {
529 use super::*;
530
531 #[test]
532 fn unsafe_package_paths_are_refused() {
533 assert!(safe_relative("post.html").is_some());
534 assert!(safe_relative("blocks/cta.html").is_some());
535 for bad in ["", "../etc/passwd", "/etc/passwd", "blocks/../../x", "./x"] {
536 assert!(safe_relative(bad).is_none(), "{bad}");
537 }
538 }
539
540 #[test]
541 fn exported_content_drops_ids_of_this_site() {
542 let schema = json!({ "nodes": [
543 { "kind": "field", "key": "cover", "widget": "mediapicker" },
544 { "kind": "field", "key": "title", "widget": "text" },
545 { "kind": "flex", "key": "body", "allowed": ["image", "gallery"], "groups": [
546 { "key": "gallery", "children": [ { "kind": "field", "key": "image_1", "widget": "mediapicker" } ] } ] }
547 ]});
548 let keys = reference_fields(&schema);
549 let mut data = json!({
550 "cover": 12, "title": "Hi",
551 "body": [
552 { "type": "image", "value": { "media_id": 3, "caption": "c" } },
553 { "type": "gallery", "value": { "image_1": 4 } }
554 ]
555 });
556 strip_references(&mut data, Some(&keys));
557 assert_eq!(data["cover"], Value::Null);
558 assert_eq!(data["title"], "Hi");
559 assert_eq!(data["body"][0]["value"]["media_id"], Value::Null);
560 assert_eq!(data["body"][0]["value"]["caption"], "c");
561 assert_eq!(data["body"][1]["value"]["image_1"], Value::Null);
562 }
563}