mars_schema/
prepare.rs

1//! One-pass package classification, normalization, selection, and strict reading.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::path::{Path, PathBuf};
5
6use serde::{Deserialize, Serialize};
7
8use crate::diagnostic::{PathSegment, SchemaDiagnostic, Severity};
9use crate::dialect::{Dialect, DialectChoice};
10use crate::discover::{SelectedDefinition, payload_paths};
11use crate::dispatch::{AsPlugin, Dispatched, Layout, dispatch};
12use crate::frontmatter::Frontmatter;
13use crate::item::DefinitionKind;
14use crate::read::{DefinitionRead, Reading, read_definition};
15use crate::tree::TreeSource;
16use crate::upgrade::UpgradeResult;
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "kebab-case")]
20pub enum PrepareContext {
21    /// A package root (a dependency, or the package being published): its `mars.toml` must
22    /// declare `[package]`, and a root with no manifest is read as the dialect it looks like.
23    Package,
24    /// A consumer's own authored directory (`.mars-src`): content without a manifest is Mars
25    /// convention, and a `mars.toml` without `[package]` is consumer configuration, not read.
26    AuthoredLocal,
27    /// A directory that is either: `Package` unless its `mars.toml` has no `[package]`, which
28    /// makes it a consumer project whose tree is read as `AuthoredLocal` content.
29    Project,
30}
31
32/// The directory below a consumer project that holds its own authored content.
33pub const LOCAL_SOURCE_DIR: &str = ".mars-src";
34
35/// Where a project directory's authored content lives and how it is read, from its `mars.toml`
36/// bytes: the project root as a `Package`, or, when the manifest is consumer configuration with
37/// no `[package]`, its `.mars-src` as `AuthoredLocal`. A caller that writes to the authored
38/// files, as `mars migrate` does, asks here instead of parsing `mars.toml` itself.
39pub fn project_authored_root(project: &Path, manifest: Option<&[u8]>) -> (PathBuf, PrepareContext) {
40    match crate::package::classify_manifest(manifest) {
41        crate::package::ManifestClassification::ConsumerConfig => (
42            project.join(LOCAL_SOURCE_DIR),
43            PrepareContext::AuthoredLocal,
44        ),
45        _ => (project.to_path_buf(), PrepareContext::Package),
46    }
47}
48
49/// What a package root is, once its manifest, declared dialect and contents have been weighed.
50#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
51#[serde(tag = "kind", rename_all = "kebab-case")]
52pub enum Classification {
53    #[serde(rename = "schema-1")]
54    Schema1,
55    MarsLegacy,
56    SchemaNewer {
57        schema: u64,
58    },
59    Invalid,
60    /// Another harness's layout, read as the dialect chosen.
61    Foreign(DialectChoice),
62    /// A Claude marketplace: it lists plugins and holds no definitions of its own.
63    Marketplace,
64    Empty,
65}
66
67#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
68#[serde(tag = "kind", rename_all = "kebab-case")]
69pub enum ChangeRecord {
70    None,
71    /// The file has no source: `normalize` wrote it (a package's generated `mars.toml`).
72    Generated,
73    Edited {
74        paths: Vec<Vec<PathSegment>>,
75    },
76    FullyRewritten,
77}
78
79#[derive(Debug, Clone, Serialize)]
80pub struct PreparedManifest {
81    /// `None` for a generated manifest.
82    #[serde(skip_serializing_if = "Option::is_none")]
83    pub source_path: Option<String>,
84    pub output_path: String,
85    /// The normalized file; crosses to JavaScript as a `Uint8Array`.
86    #[serde(with = "serde_bytes")]
87    pub normalized_bytes: Vec<u8>,
88    pub reading: DefinitionRead<crate::package::PackageMetadata>,
89    pub change: ChangeRecord,
90}
91
92#[derive(Debug, Clone, Serialize)]
93pub struct PreparedDefinition {
94    pub kind: DefinitionKind,
95    pub slug: String,
96    pub source_path: String,
97    pub output_path: String,
98    /// The normalized file; crosses to JavaScript as a `Uint8Array`.
99    #[serde(with = "serde_bytes")]
100    pub normalized_bytes: Vec<u8>,
101    /// The Agent or skill reading `kind` names.
102    pub reading: Reading,
103    pub change: ChangeRecord,
104}
105
106#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
107pub struct PayloadEntry {
108    pub source_path: String,
109    pub output_path: String,
110}
111
112#[derive(Debug, Clone, Serialize)]
113pub struct Prepared {
114    pub classification: Classification,
115    #[serde(skip_serializing_if = "Option::is_none")]
116    pub manifest: Option<PreparedManifest>,
117    pub definitions: Vec<PreparedDefinition>,
118    pub payload: Vec<PayloadEntry>,
119    /// Package-level findings by source path: what discovery and translation met and could not
120    /// convert. A definition's own reading diagnostics live on its `reading` only; see
121    /// [`Prepared::all_diagnostics`].
122    pub diagnostics: BTreeMap<String, Vec<SchemaDiagnostic>>,
123    /// What preparing changed in each file, by source path: the keys a pre-v1 upgrade or a
124    /// foreign translation dropped, moved or repaired. A caller that only reads the package
125    /// reports [`Prepared::changed_files`]; `mars migrate` lists these.
126    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
127    pub changes: BTreeMap<String, Vec<SchemaDiagnostic>>,
128}
129
130impl Prepared {
131    /// The changes that lost a restriction (a `deny` or `ask` tool entry, `permission`,
132    /// `sandbox`, `approval`), by source path. Unlike the rest of `changes`, a caller reports
133    /// these on every run: each leaves an agent less restricted than its file says.
134    pub fn lost_restrictions(&self) -> impl Iterator<Item = (&str, &SchemaDiagnostic)> {
135        self.changes.iter().flat_map(|(path, changes)| {
136            changes
137                .iter()
138                .filter(|change| change.code == crate::set_aside::RESTRICTION_LOST)
139                .map(move |change| (path.as_str(), change))
140        })
141    }
142
143    /// How many authored files preparation changed: the prepared bytes differ from the file, and
144    /// for a pre-v1 package `mars migrate` writes them back. A generated `mars.toml` has no file
145    /// to change and is not counted.
146    pub fn changed_files(&self) -> usize {
147        self.manifest
148            .iter()
149            .map(|manifest| &manifest.change)
150            .chain(self.definitions.iter().map(|definition| &definition.change))
151            .filter(|change| !matches!(change, ChangeRecord::None | ChangeRecord::Generated))
152            .count()
153    }
154
155    /// Every diagnostic by source path: package-level first, then the file's reading.
156    pub fn all_diagnostics(&self) -> BTreeMap<&str, Vec<&SchemaDiagnostic>> {
157        let mut all: BTreeMap<&str, Vec<&SchemaDiagnostic>> = BTreeMap::new();
158        for (path, diagnostics) in &self.diagnostics {
159            all.entry(path).or_default().extend(diagnostics);
160        }
161        if let Some(manifest) = &self.manifest
162            && let Some(path) = &manifest.source_path
163        {
164            all.entry(path)
165                .or_default()
166                .extend(&manifest.reading.diagnostics);
167        }
168        for definition in &self.definitions {
169            all.entry(&definition.source_path)
170                .or_default()
171                .extend(definition.reading.diagnostics());
172        }
173        all.retain(|_, diagnostics| !diagnostics.is_empty());
174        all
175    }
176}
177
178/// What a caller adds when it reads a root as one plugin of a marketplace, or imports only some
179/// of the items.
180#[derive(Default, Clone, Copy)]
181pub(crate) struct RootOptions<'a> {
182    pub as_plugin: Option<AsPlugin<'a>>,
183    /// Source paths of the definitions to import; `None` imports every one found.
184    pub selection: Option<&'a BTreeSet<String>>,
185    /// Names a root `SKILL.md` when its own `name` cannot.
186    pub fallback_name: Option<&'a str>,
187}
188
189/// `prepare` plus what was discovered, for `inspect` and `normalize`.
190pub(crate) struct Rooted {
191    pub prepared: Prepared,
192    pub dispatched: Dispatched,
193}
194
195/// Prepare one rooted package without reading payload bytes.
196pub fn prepare(
197    source: &impl TreeSource,
198    context: PrepareContext,
199    declared_dialect: Option<Dialect>,
200) -> Prepared {
201    prepare_root(source, context, declared_dialect, RootOptions::default()).prepared
202}
203
204pub(crate) fn prepare_root(
205    source: &impl TreeSource,
206    context: PrepareContext,
207    declared_dialect: Option<Dialect>,
208    options: RootOptions<'_>,
209) -> Rooted {
210    let mut dispatched = dispatch(
211        source,
212        context,
213        declared_dialect,
214        options.as_plugin,
215        options.fallback_name,
216    );
217    let classification = dispatched.classification.clone();
218    let manifest_bytes = dispatched.manifest.take();
219    let mut diagnostics = std::mem::take(&mut dispatched.discovery.diagnostics);
220    let mut changes = BTreeMap::new();
221
222    let mut manifest = manifest_bytes.as_deref().map(|bytes| {
223        prepare_manifest(
224            bytes,
225            classification == Classification::MarsLegacy,
226            &mut diagnostics,
227            &mut changes,
228        )
229    });
230
231    let can_read_definitions = !matches!(
232        classification,
233        Classification::Invalid
234            | Classification::SchemaNewer { .. }
235            | Classification::Empty
236            | Classification::Marketplace
237    );
238    let all = &dispatched.discovery.definitions;
239    let selected = all
240        .iter()
241        .filter(|definition| {
242            options
243                .selection
244                .is_none_or(|selection| selection.contains(&definition.source_path))
245        })
246        .collect::<Vec<_>>();
247    let selected_paths = selected
248        .iter()
249        .map(|definition| definition.source_path.as_str())
250        .collect::<BTreeSet<_>>();
251    let definitions = if can_read_definitions {
252        selected
253            .iter()
254            .map(|definition| {
255                prepare_definition(
256                    source,
257                    definition,
258                    &classification,
259                    all,
260                    &selected_paths,
261                    &mut diagnostics,
262                    &mut changes,
263                )
264            })
265            .collect::<Vec<_>>()
266    } else {
267        Vec::new()
268    };
269
270    let payload = if can_read_definitions {
271        payload_paths(
272            source,
273            &dispatched.discovery,
274            matches!(classification, Classification::Foreign(_)),
275            selected.iter().copied(),
276            &mut diagnostics,
277        )
278        .into_iter()
279        .map(|(source_path, output_path)| PayloadEntry {
280            source_path,
281            output_path,
282        })
283        .collect()
284    } else {
285        Vec::new()
286    };
287
288    if can_read_definitions {
289        report_unconverted(source, &dispatched, &mut diagnostics);
290    }
291    diagnostics.retain(|_, diagnostics| !diagnostics.is_empty());
292    changes.retain(|_, diagnostics| !diagnostics.is_empty());
293    let prepared = Prepared {
294        classification,
295        manifest: manifest.take(),
296        definitions,
297        payload,
298        diagnostics,
299        changes,
300    };
301    Rooted {
302        prepared,
303        dispatched,
304    }
305}
306
307/// Commands are dropped, and hooks and MCP servers are not converted yet.
308fn report_unconverted(
309    source: &impl TreeSource,
310    dispatched: &Dispatched,
311    diagnostics: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
312) {
313    let Layout::Claude(spec) = &dispatched.layout else {
314        return;
315    };
316    let mut warn = |path: &str, code: &str, message: String| {
317        diagnostics
318            .entry(path.to_owned())
319            .or_default()
320            .push(SchemaDiagnostic::new(
321                code,
322                Severity::Warning,
323                Vec::new(),
324                message,
325            ));
326    };
327    if !dispatched.discovery.dropped.is_empty() {
328        warn(
329            "commands",
330            "import.commands-dropped",
331            format!(
332                "commands are not imported (convert one to a skill): {}",
333                dispatched
334                    .discovery
335                    .dropped
336                    .iter()
337                    .map(|command| command.source_path.as_str())
338                    .collect::<Vec<_>>()
339                    .join(", ")
340            ),
341        );
342    }
343    if source.is_file("hooks/hooks.json") || spec.hooks {
344        let path = if source.is_file("hooks/hooks.json") {
345            "hooks/hooks.json"
346        } else {
347            crate::claude::manifest::PLUGIN_MANIFEST
348        };
349        warn(
350            path,
351            "import.hooks-not-converted",
352            "hooks are not converted yet".into(),
353        );
354    }
355    if source.is_file(".mcp.json") || spec.mcp {
356        let path = if source.is_file(".mcp.json") {
357            ".mcp.json"
358        } else {
359            crate::claude::manifest::PLUGIN_MANIFEST
360        };
361        warn(
362            path,
363            "import.mcp-not-converted",
364            "MCP servers are not converted yet".into(),
365        );
366    }
367}
368
369fn prepare_manifest(
370    bytes: &[u8],
371    upgrade: bool,
372    diagnostics: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
373    changes: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
374) -> PreparedManifest {
375    let invalid_utf8 = std::str::from_utf8(bytes).err().map(|error| {
376        SchemaDiagnostic::new(
377            "text.invalid-utf8",
378            Severity::Error,
379            Vec::new(),
380            format!("mars.toml is not UTF-8: {error}"),
381        )
382    });
383    let (normalized_bytes, change, upgraded) = match std::str::from_utf8(bytes) {
384        Ok(text) if upgrade => match crate::upgrade::upgrade_manifest(text) {
385            Ok(result) => upgrade_result(text, result),
386            Err(error) => {
387                push(
388                    diagnostics,
389                    "mars.toml",
390                    SchemaDiagnostic::new(
391                        "manifest.upgrade-failed",
392                        Severity::Error,
393                        Vec::new(),
394                        error.to_string(),
395                    ),
396                );
397                (bytes.to_vec(), ChangeRecord::None, Vec::new())
398            }
399        },
400        Ok(_) => (bytes.to_vec(), ChangeRecord::None, Vec::new()),
401        Err(_) => (bytes.to_vec(), ChangeRecord::None, Vec::new()),
402    };
403    changes
404        .entry("mars.toml".to_string())
405        .or_default()
406        .extend(upgraded);
407    let reading = std::str::from_utf8(&normalized_bytes).map_or_else(
408        |_| DefinitionRead::unread(invalid_utf8.into_iter().collect()),
409        crate::manifest::read,
410    );
411    PreparedManifest {
412        source_path: Some("mars.toml".to_string()),
413        output_path: "mars.toml".to_string(),
414        normalized_bytes,
415        reading,
416        change,
417    }
418}
419
420fn prepare_definition(
421    source: &impl TreeSource,
422    selected: &SelectedDefinition,
423    classification: &Classification,
424    all: &[SelectedDefinition],
425    selected_paths: &BTreeSet<&str>,
426    diagnostics: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
427    changes: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
428) -> PreparedDefinition {
429    let Some(bytes) = source.read(&selected.source_path) else {
430        return PreparedDefinition {
431            kind: selected.kind,
432            slug: selected.slug.clone(),
433            source_path: selected.source_path.clone(),
434            output_path: selected.output_path.clone(),
435            normalized_bytes: Vec::new(),
436            reading: unreadable(
437                selected.kind,
438                SchemaDiagnostic::new(
439                    "definition.read-missing",
440                    Severity::Error,
441                    Vec::new(),
442                    "selected definition could not be read",
443                ),
444            ),
445            change: ChangeRecord::None,
446        };
447    };
448    let (mut notes, mut findings) = (Vec::new(), Vec::new());
449    let (normalized_bytes, change) = match std::str::from_utf8(&bytes) {
450        Ok(text) => match classification {
451            Classification::MarsLegacy => {
452                match crate::upgrade::upgrade_definition(selected.kind, text) {
453                    Ok(result) => {
454                        let (upgraded, change, upgrade_notes) = upgrade_result(text, result);
455                        notes = upgrade_notes;
456                        (upgraded, change)
457                    }
458                    Err(error) => {
459                        push(
460                            diagnostics,
461                            &selected.source_path,
462                            SchemaDiagnostic::new(
463                                "definition.upgrade-failed",
464                                Severity::Error,
465                                Vec::new(),
466                                error.to_string(),
467                            ),
468                        );
469                        (bytes.clone(), ChangeRecord::None)
470                    }
471                }
472            }
473            Classification::Foreign(choice) => {
474                let translated =
475                    translate_foreign(text, choice.dialect, selected, all, selected_paths, &bytes);
476                notes = translated.changes;
477                findings = translated.findings;
478                (translated.bytes, translated.change)
479            }
480            _ => (bytes.clone(), ChangeRecord::None),
481        },
482        Err(_) => (bytes.clone(), ChangeRecord::None),
483    };
484    changes
485        .entry(selected.source_path.clone())
486        .or_default()
487        .extend(notes);
488    diagnostics
489        .entry(selected.source_path.clone())
490        .or_default()
491        .extend(findings);
492
493    // Only text is upgraded or translated, so these bytes are the file's when they are not text.
494    let reading = match std::str::from_utf8(&normalized_bytes) {
495        Ok(text) => read_definition(selected.kind, text),
496        Err(error) => unreadable(
497            selected.kind,
498            SchemaDiagnostic::new(
499                "text.invalid-utf8",
500                Severity::Error,
501                Vec::new(),
502                format!("definition is not UTF-8: {error}"),
503            ),
504        ),
505    };
506
507    PreparedDefinition {
508        kind: selected.kind,
509        slug: selected.slug.clone(),
510        source_path: selected.source_path.clone(),
511        output_path: selected.output_path.clone(),
512        normalized_bytes,
513        reading,
514        change,
515    }
516}
517
518/// What translating one foreign definition produced.
519struct Translated {
520    bytes: Vec<u8>,
521    change: ChangeRecord,
522    /// What the translation dropped, moved or repaired.
523    changes: Vec<SchemaDiagnostic>,
524    /// What it met and could not convert.
525    findings: Vec<SchemaDiagnostic>,
526}
527
528impl Translated {
529    fn untouched(bytes: &[u8]) -> Self {
530        Self {
531            bytes: bytes.to_vec(),
532            change: ChangeRecord::None,
533            changes: Vec::new(),
534            findings: Vec::new(),
535        }
536    }
537}
538
539/// Lift a foreign definition's frontmatter into schema v1. Frontmatter that cannot be read
540/// stays as written, with the reader's error; only a Claude skill's unquoted `description`
541/// colon is repaired first.
542fn translate_foreign(
543    text: &str,
544    dialect: Dialect,
545    selected: &SelectedDefinition,
546    all: &[SelectedDefinition],
547    selected_paths: &BTreeSet<&str>,
548    bytes: &[u8],
549) -> Translated {
550    let claude = matches!(dialect, Dialect::Claude | Dialect::AgentSkills);
551    let mut changes = Vec::new();
552    let mut repaired = false;
553    let frontmatter = match Frontmatter::parse(text) {
554        Ok(frontmatter) => frontmatter,
555        Err(_) if claude && selected.kind == DefinitionKind::Skill => {
556            match crate::claude::translate::repair_description(text)
557                .and_then(|fixed| Frontmatter::parse(&fixed).ok())
558            {
559                Some(frontmatter) => {
560                    repaired = true;
561                    changes.push(crate::claude::translate::repaired_diagnostic());
562                    frontmatter
563                }
564                None => return Translated::untouched(bytes),
565            }
566        }
567        Err(_) => return Translated::untouched(bytes),
568    };
569    let lifted = crate::lift::lift_frontmatter(dialect, selected.kind, &frontmatter);
570    changes.extend(lifted.changes);
571    let mut findings = lifted.findings;
572    if claude && selected.kind == DefinitionKind::Skill {
573        findings.extend(crate::claude::checks::skill_checks(
574            selected,
575            &lifted.frontmatter,
576            all,
577            selected_paths,
578        ));
579    }
580    let (bytes, change) = if lifted.changed || repaired {
581        (
582            lifted.frontmatter.render().into_bytes(),
583            ChangeRecord::FullyRewritten,
584        )
585    } else {
586        (bytes.to_vec(), ChangeRecord::None)
587    };
588    Translated {
589        bytes,
590        change,
591        changes,
592        findings,
593    }
594}
595
596/// A definition with no fields to read, only why.
597fn unreadable(kind: DefinitionKind, diagnostic: SchemaDiagnostic) -> Reading {
598    match kind {
599        DefinitionKind::Agent => Reading::Agent(DefinitionRead::unread(vec![diagnostic])),
600        DefinitionKind::Skill => Reading::Skill(DefinitionRead::unread(vec![diagnostic])),
601    }
602}
603
604fn upgrade_result(
605    original: &str,
606    result: UpgradeResult,
607) -> (Vec<u8>, ChangeRecord, Vec<SchemaDiagnostic>) {
608    let change = if !result.changed_from(original) {
609        ChangeRecord::None
610    } else if result.fully_rewritten {
611        ChangeRecord::FullyRewritten
612    } else {
613        ChangeRecord::Edited {
614            paths: result.edited_paths,
615        }
616    };
617    (result.text.into_bytes(), change, result.diagnostics)
618}
619
620fn push(
621    diagnostics: &mut BTreeMap<String, Vec<SchemaDiagnostic>>,
622    path: &str,
623    diagnostic: SchemaDiagnostic,
624) {
625    diagnostics
626        .entry(path.to_owned())
627        .or_default()
628        .push(diagnostic);
629}
mars_schema/prepare.rs

mars_schema/
prepare.rs
