skippy_package_format/
lib.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::fmt;
3use std::path::{Component, Path};
4
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use sha2::{Digest, Sha256};
8
9pub mod materialization;
10pub mod stage_admission;
11
12pub const PACKAGE_SCHEMA_VERSION: u32 = 2;
13
14#[derive(Debug, Clone, PartialEq)]
15pub struct PackageManifest {
16    pub schema_version: u32,
17    pub package_id: String,
18    pub model_id: String,
19    pub source_model: SourceModel,
20    pub format: String,
21    pub layer_count: u32,
22    pub model_metadata: BTreeMap<String, Value>,
23    pub artifact_catalog: ArtifactCatalog,
24    pub tensor_catalog: TensorCatalog,
25    pub sidecars: Vec<Sidecar>,
26    pub generation: Option<Generation>,
27    pub native_abi_version: String,
28    pub generator_version: String,
29    pub created_at_unix_secs: u64,
30}
31
32#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
33#[serde(deny_unknown_fields)]
34struct PackageRoot {
35    schema_version: u32,
36    package_id: String,
37    model_id: String,
38    source_model: SourceModel,
39    format: String,
40    layer_count: u32,
41    artifact_catalog: ArtifactCatalog,
42    #[serde(default, skip_serializing_if = "Vec::is_empty")]
43    sidecars: Vec<Sidecar>,
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    generation: Option<Generation>,
46    native_abi_version: String,
47    generator_version: String,
48    created_at_unix_secs: u64,
49}
50
51impl From<&PackageManifest> for PackageRoot {
52    fn from(manifest: &PackageManifest) -> Self {
53        Self {
54            schema_version: manifest.schema_version,
55            package_id: manifest.package_id.clone(),
56            model_id: manifest.model_id.clone(),
57            source_model: manifest.source_model.clone(),
58            format: manifest.format.clone(),
59            layer_count: manifest.layer_count,
60            artifact_catalog: manifest.artifact_catalog.clone(),
61            sidecars: manifest.sidecars.clone(),
62            generation: manifest.generation.clone(),
63            native_abi_version: manifest.native_abi_version.clone(),
64            generator_version: manifest.generator_version.clone(),
65            created_at_unix_secs: manifest.created_at_unix_secs,
66        }
67    }
68}
69
70impl PackageRoot {
71    fn into_unresolved(self) -> PackageManifest {
72        PackageManifest {
73            schema_version: self.schema_version,
74            package_id: self.package_id,
75            model_id: self.model_id,
76            source_model: self.source_model,
77            format: self.format,
78            layer_count: self.layer_count,
79            model_metadata: BTreeMap::new(),
80            artifact_catalog: self.artifact_catalog,
81            tensor_catalog: TensorCatalog {
82                entries: Vec::new(),
83            },
84            sidecars: self.sidecars,
85            generation: self.generation,
86            native_abi_version: self.native_abi_version,
87            generator_version: self.generator_version,
88            created_at_unix_secs: self.created_at_unix_secs,
89        }
90    }
91}
92
93impl Serialize for PackageManifest {
94    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
95    where
96        S: serde::Serializer,
97    {
98        PackageRoot::from(self).serialize(serializer)
99    }
100}
101
102impl<'de> Deserialize<'de> for PackageManifest {
103    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
104    where
105        D: serde::Deserializer<'de>,
106    {
107        PackageRoot::deserialize(deserializer).map(PackageRoot::into_unresolved)
108    }
109}
110
111impl PackageManifest {
112    /// Validate manifest structure and self-consistency, not source completeness.
113    /// Creation/certification must separately compare with independent source
114    /// directories and verify the artifact bytes; a manifest cannot certify itself.
115    pub fn validate(&self) -> Result<(), ValidationErrors> {
116        self.validate_with_inventory(true)
117    }
118
119    /// Validate the small on-disk root before its metadata carrier is opened.
120    pub fn validate_root(&self) -> Result<(), ValidationErrors> {
121        self.validate_with_inventory(false)
122    }
123
124    fn validate_with_inventory(&self, require_inventory: bool) -> Result<(), ValidationErrors> {
125        let mut issues = Vec::new();
126        validate_manifest_fields(self, require_inventory, &mut issues);
127
128        let artifacts = collect_artifacts(&self.artifact_catalog.entries, &mut issues);
129        validate_metadata_artifact_binding(self, &artifacts, &mut issues);
130        validate_sidecars(&self.sidecars, &artifacts, &mut issues);
131        if let Some(generation) = &self.generation {
132            validate_generation(generation, self.layer_count, &mut issues);
133        }
134        if require_inventory {
135            let tensors = collect_tensors(self, &mut issues);
136            validate_owned_tensor_storage(&self.tensor_catalog.entries, &artifacts, &mut issues);
137            validate_aliases(&self.tensor_catalog.entries, &tensors, &mut issues);
138        }
139
140        if issues.is_empty() {
141            Ok(())
142        } else {
143            Err(ValidationErrors { issues })
144        }
145    }
146
147    pub fn computed_package_id(&self) -> Result<String, serde_json::Error> {
148        let mut normalized = PackageRoot::from(self);
149        normalized.package_id.clear();
150        normalized.created_at_unix_secs = 0;
151        normalized
152            .source_model
153            .files
154            .sort_by(|left, right| left.path.cmp(&right.path));
155        normalized
156            .artifact_catalog
157            .entries
158            .sort_by(|left, right| left.id.cmp(&right.id));
159        normalized.sidecars.sort();
160        let digest = Sha256::digest(serde_json::to_vec(&normalized)?);
161        let hex = digest
162            .iter()
163            .map(|byte| format!("{byte:02x}"))
164            .collect::<String>();
165        Ok(format!("sha256:{hex}"))
166    }
167
168    /// Attach the inventory derived from the verified metadata carrier.
169    pub fn resolve_inventory(
170        mut self,
171        model_metadata: BTreeMap<String, Value>,
172        tensor_catalog: TensorCatalog,
173    ) -> Self {
174        self.model_metadata = model_metadata;
175        self.tensor_catalog = tensor_catalog;
176        self
177    }
178}
179
180#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
181#[serde(deny_unknown_fields)]
182pub struct SourceModel {
183    /// Digest of the primary file in the immutable source distribution.
184    pub sha256: String,
185    /// Generated package artifact that carries the model metadata needed to
186    /// construct the metadata-only graph.
187    pub metadata_artifact_id: String,
188    #[serde(default, skip_serializing_if = "Option::is_none")]
189    pub repo: Option<String>,
190    #[serde(default, skip_serializing_if = "Option::is_none")]
191    pub revision: Option<String>,
192    #[serde(default, skip_serializing_if = "Option::is_none")]
193    pub primary_file: Option<String>,
194    #[serde(default, skip_serializing_if = "Option::is_none")]
195    pub canonical_ref: Option<String>,
196    #[serde(default, skip_serializing_if = "Option::is_none")]
197    pub distribution_id: Option<String>,
198    #[serde(default, skip_serializing_if = "Vec::is_empty")]
199    pub files: Vec<SourceFile>,
200}
201
202#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
203#[serde(deny_unknown_fields)]
204pub struct SourceFile {
205    pub path: String,
206    pub byte_size: u64,
207    pub sha256: String,
208}
209
210#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
211#[serde(deny_unknown_fields)]
212pub struct ArtifactCatalog {
213    pub entries: Vec<Artifact>,
214}
215
216#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
217#[serde(deny_unknown_fields)]
218pub struct Artifact {
219    pub id: String,
220    pub path: String,
221    pub byte_size: u64,
222    pub sha256: String,
223}
224
225#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
226#[serde(deny_unknown_fields)]
227pub struct TensorCatalog {
228    pub entries: Vec<Tensor>,
229}
230
231#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
232#[serde(deny_unknown_fields)]
233pub struct Tensor {
234    pub id: String,
235    pub name: String,
236    pub ggml_type: u32,
237    pub dimensions: Vec<u64>,
238    #[serde(default, skip_serializing_if = "Option::is_none")]
239    pub layer_ordinal: Option<u32>,
240    pub storage: TensorStorage,
241}
242
243#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
244#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
245pub enum TensorStorage {
246    Owned {
247        artifact_id: String,
248        data_offset: u64,
249        stored_length: u64,
250        alignment: u64,
251        integrity: TensorIntegrity,
252    },
253    Alias {
254        target_tensor_id: String,
255    },
256}
257
258#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
259#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
260pub enum TensorIntegrity {
261    ArtifactSha256,
262    TensorSha256 { sha256: String },
263}
264
265#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
266#[serde(deny_unknown_fields)]
267pub struct Sidecar {
268    pub kind: SidecarKind,
269    pub artifact_id: String,
270    #[serde(default, skip_serializing_if = "Option::is_none")]
271    pub name: Option<String>,
272}
273
274impl Ord for Sidecar {
275    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
276        (&self.kind, &self.name, &self.artifact_id).cmp(&(
277            &other.kind,
278            &other.name,
279            &other.artifact_id,
280        ))
281    }
282}
283
284impl PartialOrd for Sidecar {
285    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
286        Some(self.cmp(other))
287    }
288}
289
290#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
291#[serde(rename_all = "snake_case")]
292pub enum SidecarKind {
293    Mmproj,
294}
295
296/// Generation capability declarations carried as package data.
297///
298/// Describes speculative-decoding capabilities of the source model only.
299/// It is never stage-selection or loader policy: which strategy actually runs
300/// is resolved from runtime configuration, and layer indices here must never
301/// be consulted by a stage selector.
302#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
303#[serde(deny_unknown_fields)]
304pub struct Generation {
305    #[serde(default, skip_serializing_if = "Option::is_none")]
306    pub speculative_decoding: Option<SpeculativeDecoding>,
307}
308
309#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
310#[serde(deny_unknown_fields)]
311pub struct SpeculativeDecoding {
312    pub default: String,
313    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
314    pub proposers: BTreeMap<String, ProposerSpec>,
315    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
316    pub strategies: BTreeMap<String, StrategySpec>,
317}
318
319/// Strategy wrapper: `#[serde(flatten)]` of the tagged enum forbids
320/// `deny_unknown_fields` here, so unknown-field rejection happens inside the
321/// tagged enum variants.
322#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
323pub struct ProposerSpec {
324    #[serde(flatten)]
325    pub kind: ProposerKind,
326}
327
328/// See [`ProposerSpec`] for the flatten/deny_unknown_fields tradeoff.
329#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
330pub struct StrategySpec {
331    #[serde(flatten)]
332    pub kind: StrategyKind,
333}
334
335#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
336#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)]
337pub enum ProposerKind {
338    #[serde(rename = "native-mtp")]
339    NativeMtp {
340        prediction_depth: u32,
341        layer_indices: Vec<u32>,
342    },
343    NgramCache {
344        ngram_min: u32,
345        ngram_max: u32,
346        max_proposal_tokens: u32,
347        history_scope: String,
348    },
349    NgramSuffix {
350        ngram_min: u32,
351        ngram_max: u32,
352        max_proposal_tokens: u32,
353        history_scope: String,
354    },
355}
356
357#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
358#[serde(deny_unknown_fields)]
359pub struct WindowPolicy {
360    pub default: String,
361    pub initial_window: u32,
362    pub min_window: u32,
363    pub max_window: u32,
364    #[serde(default, skip_serializing_if = "Option::is_none")]
365    pub pipeline_depth: Option<u32>,
366}
367
368#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
369#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)]
370pub enum StrategyKind {
371    /// Native-MTP strategies carry either a proposer reference or the inline
372    /// `prediction_depth`/`layer_indices` form — never both, never neither.
373    #[serde(rename = "native-mtp")]
374    NativeMtp {
375        #[serde(default, skip_serializing_if = "Option::is_none")]
376        proposer: Option<String>,
377        #[serde(default, skip_serializing_if = "Option::is_none")]
378        prediction_depth: Option<u32>,
379        #[serde(default, skip_serializing_if = "Vec::is_empty")]
380        layer_indices: Vec<u32>,
381        #[serde(default, skip_serializing_if = "Option::is_none")]
382        window_policy: Option<WindowPolicy>,
383    },
384    NgramCache {
385        proposer: String,
386        #[serde(default, skip_serializing_if = "Option::is_none")]
387        window_policy: Option<WindowPolicy>,
388    },
389    NgramSuffix {
390        proposer: String,
391        #[serde(default, skip_serializing_if = "Option::is_none")]
392        window_policy: Option<WindowPolicy>,
393    },
394    Composite {
395        primary: String,
396        extender: String,
397        extension_policy: ExtensionPolicy,
398        #[serde(default, skip_serializing_if = "Option::is_none")]
399        window_policy: Option<WindowPolicy>,
400    },
401}
402
403#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
404#[serde(deny_unknown_fields)]
405pub struct ExtensionPolicy {
406    pub max_tokens: u32,
407    #[serde(default, skip_serializing_if = "Option::is_none")]
408    pub initial_tokens: Option<u32>,
409    #[serde(default, skip_serializing_if = "Option::is_none")]
410    pub tail_backoff_proposals: Option<u32>,
411}
412
413#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
414pub struct ValidationIssue {
415    pub code: ValidationCode,
416    pub path: String,
417    pub message: String,
418}
419
420#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
421#[serde(rename_all = "snake_case")]
422pub enum ValidationCode {
423    UnsupportedSchemaVersion,
424    MissingValue,
425    InvalidDigest,
426    InvalidIdentifier,
427    InvalidPath,
428    UnknownSourceFile,
429    InvalidMetadataArtifact,
430    SourceIdentityMismatch,
431    DuplicateArtifact,
432    DuplicateArtifactPath,
433    DuplicateSidecar,
434    DuplicateTensor,
435    LayerOutOfBounds,
436    InvalidDimension,
437    UnknownArtifact,
438    InvalidAlignment,
439    MisalignedOffset,
440    StorageOutOfBounds,
441    InvalidStoredLength,
442    OverlappingStorage,
443    UnknownAliasTarget,
444    AliasTargetIsAlias,
445    AliasMetadataMismatch,
446    PackageIdentityMismatch,
447    UnknownStrategy,
448    UnknownProposer,
449    InvalidGenerationValue,
450}
451
452#[derive(Debug, Clone, PartialEq, Eq)]
453pub struct ValidationErrors {
454    issues: Vec<ValidationIssue>,
455}
456
457impl ValidationErrors {
458    pub fn issues(&self) -> &[ValidationIssue] {
459        &self.issues
460    }
461}
462
463impl fmt::Display for ValidationErrors {
464    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
465        write!(formatter, "invalid package manifest")?;
466        for issue in &self.issues {
467            write!(formatter, "; {}: {}", issue.path, issue.message)?;
468        }
469        Ok(())
470    }
471}
472
473impl std::error::Error for ValidationErrors {}
474
475fn validate_manifest_fields(
476    manifest: &PackageManifest,
477    require_inventory: bool,
478    issues: &mut Vec<ValidationIssue>,
479) {
480    if manifest.schema_version != PACKAGE_SCHEMA_VERSION {
481        push_issue(
482            issues,
483            ValidationCode::UnsupportedSchemaVersion,
484            "schema_version",
485            format!(
486                "expected schema version {PACKAGE_SCHEMA_VERSION}, got {}",
487                manifest.schema_version
488            ),
489        );
490    }
491    validate_nonempty("model_id", &manifest.model_id, issues);
492    validate_nonempty("format", &manifest.format, issues);
493    validate_nonempty("native_abi_version", &manifest.native_abi_version, issues);
494    validate_nonempty("generator_version", &manifest.generator_version, issues);
495    if manifest.layer_count == 0 {
496        push_issue(
497            issues,
498            ValidationCode::MissingValue,
499            "layer_count",
500            "layer count must be greater than zero",
501        );
502    }
503    validate_prefixed_sha256("package_id", &manifest.package_id, issues);
504    validate_sha256("source_model.sha256", &manifest.source_model.sha256, issues);
505    validate_nonempty(
506        "source_model.metadata_artifact_id",
507        &manifest.source_model.metadata_artifact_id,
508        issues,
509    );
510    if require_inventory && manifest.model_metadata.is_empty() {
511        push_issue(
512            issues,
513            ValidationCode::MissingValue,
514            "model_metadata",
515            "model metadata must not be empty",
516        );
517    }
518    if require_inventory && !manifest.model_metadata.contains_key("general.architecture") {
519        push_issue(
520            issues,
521            ValidationCode::MissingValue,
522            "model_metadata.general.architecture",
523            "model metadata must contain general.architecture",
524        );
525    }
526    if manifest.source_model.files.is_empty() {
527        push_issue(
528            issues,
529            ValidationCode::MissingValue,
530            "source_model.files",
531            "source file inventory must not be empty",
532        );
533    }
534    if manifest.artifact_catalog.entries.is_empty() {
535        push_issue(
536            issues,
537            ValidationCode::MissingValue,
538            "artifact_catalog.entries",
539            "artifact catalog must not be empty",
540        );
541    }
542    if require_inventory && manifest.tensor_catalog.entries.is_empty() {
543        push_issue(
544            issues,
545            ValidationCode::MissingValue,
546            "tensor_catalog.entries",
547            "tensor catalog must not be empty",
548        );
549    }
550    match manifest.computed_package_id() {
551        Ok(expected) if manifest.package_id != expected => push_issue(
552            issues,
553            ValidationCode::PackageIdentityMismatch,
554            "package_id",
555            format!(
556                "package identity {:?} does not match canonical identity {:?}",
557                manifest.package_id, expected
558            ),
559        ),
560        Ok(_) => {}
561        Err(error) => push_issue(
562            issues,
563            ValidationCode::PackageIdentityMismatch,
564            "package_id",
565            format!("cannot compute canonical package identity: {error}"),
566        ),
567    }
568
569    let mut source_paths = BTreeSet::new();
570    for (index, file) in manifest.source_model.files.iter().enumerate() {
571        validate_relative_path(
572            &format!("source_model.files[{index}].path"),
573            &file.path,
574            issues,
575        );
576        validate_sha256(
577            &format!("source_model.files[{index}].sha256"),
578            &file.sha256,
579            issues,
580        );
581        if !source_paths.insert(file.path.as_str()) {
582            push_issue(
583                issues,
584                ValidationCode::InvalidPath,
585                format!("source_model.files[{index}].path"),
586                format!("source file path {:?} appears more than once", file.path),
587            );
588        }
589    }
590    match &manifest.source_model.primary_file {
591        Some(primary_file) => {
592            validate_relative_path("source_model.primary_file", primary_file, issues);
593            if !source_paths.contains(primary_file.as_str()) {
594                push_issue(
595                    issues,
596                    ValidationCode::UnknownSourceFile,
597                    "source_model.primary_file",
598                    format!(
599                        "primary file {:?} is absent from source files",
600                        primary_file
601                    ),
602                );
603            }
604        }
605        None => {
606            push_issue(
607                issues,
608                ValidationCode::MissingValue,
609                "source_model.primary_file",
610                "primary source file is required",
611            );
612        }
613    }
614}
615
616fn validate_metadata_artifact_binding(
617    manifest: &PackageManifest,
618    artifacts: &BTreeMap<&str, &Artifact>,
619    issues: &mut Vec<ValidationIssue>,
620) {
621    let artifact_id = manifest.source_model.metadata_artifact_id.as_str();
622    if artifact_id.trim().is_empty() {
623        return;
624    }
625    let Some(artifact) = artifacts.get(artifact_id) else {
626        push_issue(
627            issues,
628            ValidationCode::UnknownArtifact,
629            "source_model.metadata_artifact_id",
630            format!("metadata artifact {artifact_id:?} is absent from artifact catalog"),
631        );
632        return;
633    };
634    if Path::new(&artifact.path)
635        .extension()
636        .and_then(|value| value.to_str())
637        != Some("gguf")
638    {
639        push_issue(
640            issues,
641            ValidationCode::InvalidMetadataArtifact,
642            "source_model.metadata_artifact_id",
643            format!("metadata artifact {:?} is not a GGUF artifact", artifact.id),
644        );
645    }
646    if manifest
647        .sidecars
648        .iter()
649        .any(|sidecar| sidecar.artifact_id == artifact.id)
650    {
651        push_issue(
652            issues,
653            ValidationCode::InvalidMetadataArtifact,
654            "source_model.metadata_artifact_id",
655            format!("metadata artifact {:?} has a sidecar role", artifact.id),
656        );
657    }
658    let Some(primary_path) = manifest.source_model.primary_file.as_deref() else {
659        return;
660    };
661    let Some(primary_file) = manifest
662        .source_model
663        .files
664        .iter()
665        .find(|file| file.path == primary_path)
666    else {
667        return;
668    };
669    if manifest.source_model.sha256 != primary_file.sha256 {
670        push_issue(
671            issues,
672            ValidationCode::SourceIdentityMismatch,
673            "source_model.sha256",
674            "source model digest does not match the primary source file",
675        );
676    }
677}
678
679fn collect_artifacts<'a>(
680    artifacts: &'a [Artifact],
681    issues: &mut Vec<ValidationIssue>,
682) -> BTreeMap<&'a str, &'a Artifact> {
683    let mut by_id = BTreeMap::new();
684    let mut paths = BTreeSet::new();
685    for (index, artifact) in artifacts.iter().enumerate() {
686        let prefix = format!("artifacts[{index}]");
687        validate_identifier(&format!("{prefix}.id"), &artifact.id, issues);
688        validate_relative_path(&format!("{prefix}.path"), &artifact.path, issues);
689        validate_sha256(&format!("{prefix}.sha256"), &artifact.sha256, issues);
690        if by_id.insert(artifact.id.as_str(), artifact).is_some() {
691            push_issue(
692                issues,
693                ValidationCode::DuplicateArtifact,
694                format!("{prefix}.id"),
695                format!("artifact id {:?} appears more than once", artifact.id),
696            );
697        }
698        if !paths.insert(artifact.path.as_str()) {
699            push_issue(
700                issues,
701                ValidationCode::DuplicateArtifactPath,
702                format!("{prefix}.path"),
703                format!("artifact path {:?} appears more than once", artifact.path),
704            );
705        }
706    }
707    by_id
708}
709
710fn collect_tensors<'a>(
711    manifest: &'a PackageManifest,
712    issues: &mut Vec<ValidationIssue>,
713) -> BTreeMap<&'a str, &'a Tensor> {
714    let tensors = &manifest.tensor_catalog.entries;
715    let mut by_id = BTreeMap::new();
716    let mut names = BTreeSet::new();
717    for (index, tensor) in tensors.iter().enumerate() {
718        let prefix = format!("tensors[{index}]");
719        validate_nonempty(&format!("{prefix}.id"), &tensor.id, issues);
720        validate_nonempty(&format!("{prefix}.name"), &tensor.name, issues);
721        if tensor.dimensions.is_empty() || tensor.dimensions.contains(&0) {
722            push_issue(
723                issues,
724                ValidationCode::InvalidDimension,
725                format!("{prefix}.dimensions"),
726                "tensor dimensions must be non-empty and non-zero",
727            );
728        }
729        if let Some(layer_ordinal) = tensor.layer_ordinal
730            && layer_ordinal >= manifest.layer_count
731        {
732            push_issue(
733                issues,
734                ValidationCode::LayerOutOfBounds,
735                format!("{prefix}.layer_ordinal"),
736                format!(
737                    "layer ordinal {layer_ordinal} is outside model layer count {}",
738                    manifest.layer_count
739                ),
740            );
741        }
742        if by_id.insert(tensor.id.as_str(), tensor).is_some() {
743            push_issue(
744                issues,
745                ValidationCode::DuplicateTensor,
746                format!("{prefix}.id"),
747                format!("tensor id {:?} appears more than once", tensor.id),
748            );
749        }
750        if !names.insert(tensor.name.as_str()) {
751            push_issue(
752                issues,
753                ValidationCode::DuplicateTensor,
754                format!("{prefix}.name"),
755                format!("tensor name {:?} appears more than once", tensor.name),
756            );
757        }
758    }
759    by_id
760}
761
762fn validate_owned_tensor_storage(
763    tensors: &[Tensor],
764    artifacts: &BTreeMap<&str, &Artifact>,
765    issues: &mut Vec<ValidationIssue>,
766) {
767    let mut ranges: BTreeMap<&str, Vec<(u64, u64, usize)>> = BTreeMap::new();
768    for (index, tensor) in tensors.iter().enumerate() {
769        let TensorStorage::Owned {
770            artifact_id,
771            data_offset,
772            stored_length,
773            alignment,
774            integrity,
775        } = &tensor.storage
776        else {
777            continue;
778        };
779        let prefix = format!("tensors[{index}].storage");
780        let Some(artifact) = artifacts.get(artifact_id.as_str()) else {
781            push_issue(
782                issues,
783                ValidationCode::UnknownArtifact,
784                format!("{prefix}.artifact_id"),
785                format!("artifact {:?} does not exist", artifact_id),
786            );
787            continue;
788        };
789        if !alignment.is_power_of_two() {
790            push_issue(
791                issues,
792                ValidationCode::InvalidAlignment,
793                format!("{prefix}.alignment"),
794                format!("alignment {alignment} is not a non-zero power of two"),
795            );
796        } else if data_offset % alignment != 0 {
797            push_issue(
798                issues,
799                ValidationCode::MisalignedOffset,
800                format!("{prefix}.data_offset"),
801                format!("offset {data_offset} is not aligned to {alignment} bytes"),
802            );
803        }
804        if *stored_length == 0 {
805            push_issue(
806                issues,
807                ValidationCode::InvalidStoredLength,
808                format!("{prefix}.stored_length"),
809                "owned tensor storage must not be empty",
810            );
811        }
812        match data_offset.checked_add(*stored_length) {
813            Some(end) if end <= artifact.byte_size => {
814                ranges
815                    .entry(artifact_id)
816                    .or_default()
817                    .push((*data_offset, end, index));
818            }
819            _ => push_issue(
820                issues,
821                ValidationCode::StorageOutOfBounds,
822                prefix.clone(),
823                format!(
824                    "range {data_offset}+{stored_length} exceeds artifact size {}",
825                    artifact.byte_size
826                ),
827            ),
828        }
829        if let TensorIntegrity::TensorSha256 { sha256 } = integrity {
830            validate_sha256(&format!("{prefix}.integrity.sha256"), sha256, issues);
831        }
832    }
833
834    for (artifact_id, artifact_ranges) in &mut ranges {
835        artifact_ranges.sort_unstable_by_key(|range| (range.0, range.1, range.2));
836        for pair in artifact_ranges.windows(2) {
837            let previous = pair[0];
838            let current = pair[1];
839            if current.0 < previous.1 {
840                push_issue(
841                    issues,
842                    ValidationCode::OverlappingStorage,
843                    format!("tensors[{}].storage", current.2),
844                    format!(
845                        "storage overlaps tensors[{}] in artifact {:?}",
846                        previous.2, artifact_id
847                    ),
848                );
849            }
850        }
851    }
852}
853
854fn validate_aliases(
855    tensors: &[Tensor],
856    by_id: &BTreeMap<&str, &Tensor>,
857    issues: &mut Vec<ValidationIssue>,
858) {
859    for (index, tensor) in tensors.iter().enumerate() {
860        let TensorStorage::Alias { target_tensor_id } = &tensor.storage else {
861            continue;
862        };
863        let path = format!("tensors[{index}].storage.target_tensor_id");
864        let Some(target) = by_id.get(target_tensor_id.as_str()) else {
865            push_issue(
866                issues,
867                ValidationCode::UnknownAliasTarget,
868                path,
869                format!("tensor {:?} does not exist", target_tensor_id),
870            );
871            continue;
872        };
873        if matches!(target.storage, TensorStorage::Alias { .. }) {
874            push_issue(
875                issues,
876                ValidationCode::AliasTargetIsAlias,
877                path,
878                format!("alias target {:?} is not owned storage", target_tensor_id),
879            );
880        }
881        if tensor.ggml_type != target.ggml_type || tensor.dimensions != target.dimensions {
882            push_issue(
883                issues,
884                ValidationCode::AliasMetadataMismatch,
885                format!("tensors[{index}].storage"),
886                format!(
887                    "alias metadata does not match target tensor {:?}",
888                    target_tensor_id
889                ),
890            );
891        }
892    }
893}
894
895fn validate_sidecars(
896    sidecars: &[Sidecar],
897    artifacts: &BTreeMap<&str, &Artifact>,
898    issues: &mut Vec<ValidationIssue>,
899) {
900    let mut seen = BTreeSet::new();
901    for (index, sidecar) in sidecars.iter().enumerate() {
902        let prefix = format!("sidecars[{index}]");
903        if !seen.insert((sidecar.kind, sidecar.name.as_deref())) {
904            push_issue(
905                issues,
906                ValidationCode::DuplicateSidecar,
907                prefix.clone(),
908                format!(
909                    "sidecar semantic identity ({:?}, {:?}) appears more than once",
910                    sidecar.kind, sidecar.name
911                ),
912            );
913        }
914        if !artifacts.contains_key(sidecar.artifact_id.as_str()) {
915            push_issue(
916                issues,
917                ValidationCode::UnknownArtifact,
918                format!("{prefix}.artifact_id"),
919                format!("artifact {:?} does not exist", sidecar.artifact_id),
920            );
921        }
922        if let Some(name) = &sidecar.name {
923            validate_nonempty(&format!("{prefix}.name"), name, issues);
924        }
925    }
926}
927
928fn validate_nonempty(path: &str, value: &str, issues: &mut Vec<ValidationIssue>) {
929    if value.trim().is_empty() {
930        push_issue(
931            issues,
932            ValidationCode::MissingValue,
933            path,
934            "value must not be empty",
935        );
936    }
937}
938
939fn validate_identifier(path: &str, value: &str, issues: &mut Vec<ValidationIssue>) {
940    if value.is_empty()
941        || !value
942            .bytes()
943            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.'))
944    {
945        push_issue(
946            issues,
947            ValidationCode::InvalidIdentifier,
948            path,
949            format!("identifier {:?} contains unsupported characters", value),
950        );
951    }
952}
953
954fn validate_relative_path(path: &str, value: &str, issues: &mut Vec<ValidationIssue>) {
955    let candidate = Path::new(value);
956    if value.is_empty()
957        || value.contains('\\')
958        || value.contains("//")
959        || value.ends_with('/')
960        || candidate.is_absolute()
961        || candidate
962            .components()
963            .any(|component| !matches!(component, Component::Normal(_)))
964    {
965        push_issue(
966            issues,
967            ValidationCode::InvalidPath,
968            path,
969            format!(
970                "path {:?} must be a normalized relative package path",
971                value
972            ),
973        );
974    }
975}
976
977fn validate_prefixed_sha256(path: &str, value: &str, issues: &mut Vec<ValidationIssue>) {
978    let Some(digest) = value.strip_prefix("sha256:") else {
979        push_issue(
980            issues,
981            ValidationCode::InvalidDigest,
982            path,
983            "digest must use the sha256:<hex> form",
984        );
985        return;
986    };
987    validate_sha256(path, digest, issues);
988}
989
990fn validate_sha256(path: &str, value: &str, issues: &mut Vec<ValidationIssue>) {
991    if value.len() != 64
992        || !value
993            .bytes()
994            .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))
995    {
996        push_issue(
997            issues,
998            ValidationCode::InvalidDigest,
999            path,
1000            "digest must contain 64 lowercase hexadecimal characters",
1001        );
1002    }
1003}
1004
1005fn push_issue(
1006    issues: &mut Vec<ValidationIssue>,
1007    code: ValidationCode,
1008    path: impl Into<String>,
1009    message: impl Into<String>,
1010) {
1011    issues.push(ValidationIssue {
1012        code,
1013        path: path.into(),
1014        message: message.into(),
1015    });
1016}
1017
1018/// Schema-level (graph-checkable) generation validation only.
1019///
1020/// Deliberately excludes runtime-capability bounds such as the native-MTP
1021/// prediction-depth value, n-gram window limits, or history-scope vocabulary:
1022/// those are enforced by the runtime/preflight, not by the package format.
1023fn validate_generation(
1024    generation: &Generation,
1025    layer_count: u32,
1026    issues: &mut Vec<ValidationIssue>,
1027) {
1028    let Some(speculative) = &generation.speculative_decoding else {
1029        return;
1030    };
1031    if speculative.default.trim().is_empty() {
1032        push_issue(
1033            issues,
1034            ValidationCode::InvalidGenerationValue,
1035            "generation.speculative_decoding.default",
1036            "default must not be empty",
1037        );
1038    }
1039    for name in speculative.proposers.keys() {
1040        if name.trim().is_empty() {
1041            push_issue(
1042                issues,
1043                ValidationCode::InvalidGenerationValue,
1044                "generation.speculative_decoding.proposers",
1045                "proposer names must not be empty",
1046            );
1047        }
1048        validate_proposer(name, &speculative.proposers[name], layer_count, issues);
1049    }
1050    for name in speculative.strategies.keys() {
1051        if name.trim().is_empty() {
1052            push_issue(
1053                issues,
1054                ValidationCode::InvalidGenerationValue,
1055                "generation.speculative_decoding.strategies",
1056                "strategy names must not be empty",
1057            );
1058        }
1059        validate_strategy(
1060            name,
1061            &speculative.strategies[name],
1062            &speculative.proposers,
1063            layer_count,
1064            issues,
1065        );
1066    }
1067    if !speculative.default.trim().is_empty()
1068        && !speculative.strategies.contains_key(&speculative.default)
1069    {
1070        push_issue(
1071            issues,
1072            ValidationCode::UnknownStrategy,
1073            "generation.speculative_decoding.default",
1074            format!(
1075                "default strategy {:?} is not declared under strategies",
1076                speculative.default
1077            ),
1078        );
1079    }
1080}
1081
1082fn validate_proposer(
1083    name: &str,
1084    proposer: &ProposerSpec,
1085    layer_count: u32,
1086    issues: &mut Vec<ValidationIssue>,
1087) {
1088    match &proposer.kind {
1089        ProposerKind::NativeMtp {
1090            prediction_depth,
1091            layer_indices,
1092        } => {
1093            if *prediction_depth == 0 {
1094                push_issue(
1095                    issues,
1096                    ValidationCode::InvalidGenerationValue,
1097                    format!("generation.speculative_decoding.proposers[{name}].prediction_depth"),
1098                    "prediction_depth must be greater than zero",
1099                );
1100            }
1101            if layer_indices.is_empty() {
1102                push_issue(
1103                    issues,
1104                    ValidationCode::InvalidGenerationValue,
1105                    format!("generation.speculative_decoding.proposers[{name}].layer_indices"),
1106                    "layer_indices must not be empty",
1107                );
1108            }
1109            for (index, layer_index) in layer_indices.iter().enumerate() {
1110                if *layer_index >= layer_count {
1111                    push_issue(
1112                        issues,
1113                        ValidationCode::LayerOutOfBounds,
1114                        format!(
1115                            "generation.speculative_decoding.proposers[{name}].layer_indices[{index}]"
1116                        ),
1117                        format!(
1118                            "layer index {layer_index} is out of range for layer_count {layer_count}"
1119                        ),
1120                    );
1121                }
1122            }
1123        }
1124        ProposerKind::NgramCache {
1125            ngram_min,
1126            ngram_max,
1127            max_proposal_tokens,
1128            history_scope,
1129        }
1130        | ProposerKind::NgramSuffix {
1131            ngram_min,
1132            ngram_max,
1133            max_proposal_tokens,
1134            history_scope,
1135        } => {
1136            if *ngram_min == 0 || *ngram_max < *ngram_min {
1137                push_issue(
1138                    issues,
1139                    ValidationCode::InvalidGenerationValue,
1140                    format!("generation.speculative_decoding.proposers[{name}]"),
1141                    "ngram_min and ngram_max must satisfy 1 <= ngram_min <= ngram_max",
1142                );
1143            }
1144            if *max_proposal_tokens == 0 {
1145                push_issue(
1146                    issues,
1147                    ValidationCode::InvalidGenerationValue,
1148                    format!(
1149                        "generation.speculative_decoding.proposers[{name}].max_proposal_tokens"
1150                    ),
1151                    "max_proposal_tokens must be greater than zero",
1152                );
1153            }
1154            if history_scope.trim().is_empty() {
1155                push_issue(
1156                    issues,
1157                    ValidationCode::InvalidGenerationValue,
1158                    format!("generation.speculative_decoding.proposers[{name}].history_scope"),
1159                    "history_scope must not be empty",
1160                );
1161            }
1162        }
1163    }
1164}
1165
1166fn validate_strategy(
1167    name: &str,
1168    strategy: &StrategySpec,
1169    proposers: &BTreeMap<String, ProposerSpec>,
1170    layer_count: u32,
1171    issues: &mut Vec<ValidationIssue>,
1172) {
1173    let require_proposer = |field: &str,
1174                            proposer_name: &str,
1175                            allowed: &[&str],
1176                            issues: &mut Vec<ValidationIssue>| {
1177        if proposer_name.trim().is_empty() {
1178            push_issue(
1179                issues,
1180                ValidationCode::InvalidGenerationValue,
1181                format!("generation.speculative_decoding.strategies[{name}].{field}"),
1182                "proposer reference must not be empty",
1183            );
1184        } else if !proposers.contains_key(proposer_name) {
1185            push_issue(
1186                issues,
1187                ValidationCode::UnknownProposer,
1188                format!("generation.speculative_decoding.strategies[{name}].{field}"),
1189                format!("references undeclared proposer {proposer_name:?}"),
1190            );
1191        } else {
1192            let actual = match &proposers[proposer_name].kind {
1193                ProposerKind::NativeMtp { .. } => "native-mtp",
1194                ProposerKind::NgramCache { .. } => "ngram-cache",
1195                ProposerKind::NgramSuffix { .. } => "ngram-suffix",
1196            };
1197            if !allowed.contains(&actual) {
1198                push_issue(
1199                    issues,
1200                    ValidationCode::InvalidGenerationValue,
1201                    format!("generation.speculative_decoding.strategies[{name}].{field}"),
1202                    format!(
1203                        "proposer {proposer_name:?} has kind {actual:?}, which is not compatible with this strategy (expected one of {allowed:?})"
1204                    ),
1205                );
1206            }
1207        }
1208    };
1209    let validate_window_policy =
1210        |window_policy: &Option<WindowPolicy>, issues: &mut Vec<ValidationIssue>| {
1211            if let Some(window) = window_policy {
1212                if window.default.trim().is_empty() {
1213                    push_issue(
1214                        issues,
1215                        ValidationCode::InvalidGenerationValue,
1216                        format!(
1217                            "generation.speculative_decoding.strategies[{name}].window_policy.default"
1218                        ),
1219                        "window_policy.default must not be empty",
1220                    );
1221                }
1222                if window.initial_window == 0 || window.min_window == 0 || window.max_window == 0 {
1223                    push_issue(
1224                        issues,
1225                        ValidationCode::InvalidGenerationValue,
1226                        format!("generation.speculative_decoding.strategies[{name}].window_policy"),
1227                        "window_policy windows must be greater than zero",
1228                    );
1229                }
1230                if window.pipeline_depth == Some(0) {
1231                    push_issue(
1232                        issues,
1233                        ValidationCode::InvalidGenerationValue,
1234                        format!(
1235                            "generation.speculative_decoding.strategies[{name}].window_policy.pipeline_depth"
1236                        ),
1237                        "window_policy.pipeline_depth must be greater than zero",
1238                    );
1239                }
1240                if window.min_window > window.max_window {
1241                    push_issue(
1242                        issues,
1243                        ValidationCode::InvalidGenerationValue,
1244                        format!("generation.speculative_decoding.strategies[{name}].window_policy"),
1245                        "window_policy min_window must not exceed max_window",
1246                    );
1247                }
1248                if window.initial_window < window.min_window
1249                    || window.initial_window > window.max_window
1250                {
1251                    push_issue(
1252                        issues,
1253                        ValidationCode::InvalidGenerationValue,
1254                        format!("generation.speculative_decoding.strategies[{name}].window_policy"),
1255                        "window_policy initial_window must lie within min_window..max_window",
1256                    );
1257                }
1258            }
1259        };
1260    match &strategy.kind {
1261        StrategyKind::NativeMtp {
1262            proposer,
1263            prediction_depth,
1264            layer_indices,
1265            window_policy,
1266        } => {
1267            validate_window_policy(window_policy, issues);
1268            match (
1269                proposer,
1270                prediction_depth.is_some(),
1271                !layer_indices.is_empty(),
1272            ) {
1273                (Some(proposer_name), false, false) => {
1274                    require_proposer("proposer", proposer_name, &["native-mtp"], issues);
1275                }
1276                (None, true, true) => {
1277                    if *prediction_depth == Some(0) {
1278                        push_issue(
1279                            issues,
1280                            ValidationCode::InvalidGenerationValue,
1281                            format!(
1282                                "generation.speculative_decoding.strategies[{name}].prediction_depth"
1283                            ),
1284                            "prediction_depth must be greater than zero",
1285                        );
1286                    }
1287                    for (index, layer_index) in layer_indices.iter().enumerate() {
1288                        if *layer_index >= layer_count {
1289                            push_issue(
1290                                issues,
1291                                ValidationCode::LayerOutOfBounds,
1292                                format!(
1293                                    "generation.speculative_decoding.strategies[{name}].layer_indices[{index}]"
1294                                ),
1295                                format!(
1296                                    "layer index {layer_index} is out of range for layer_count {layer_count}"
1297                                ),
1298                            );
1299                        }
1300                    }
1301                }
1302                (Some(_), _, _) => push_issue(
1303                    issues,
1304                    ValidationCode::InvalidGenerationValue,
1305                    format!("generation.speculative_decoding.strategies[{name}]"),
1306                    "native-mtp strategy must set either proposer or the inline prediction_depth/layer_indices form, not both",
1307                ),
1308                (None, true, false) => push_issue(
1309                    issues,
1310                    ValidationCode::InvalidGenerationValue,
1311                    format!("generation.speculative_decoding.strategies[{name}].layer_indices"),
1312                    "inline native-mtp strategy must declare layer_indices alongside prediction_depth",
1313                ),
1314                (None, false, true) => push_issue(
1315                    issues,
1316                    ValidationCode::InvalidGenerationValue,
1317                    format!("generation.speculative_decoding.strategies[{name}].prediction_depth"),
1318                    "inline native-mtp strategy must declare prediction_depth alongside layer_indices",
1319                ),
1320                (None, false, false) => push_issue(
1321                    issues,
1322                    ValidationCode::InvalidGenerationValue,
1323                    format!("generation.speculative_decoding.strategies[{name}]"),
1324                    "native-mtp strategy must set either proposer or the inline prediction_depth/layer_indices form",
1325                ),
1326            }
1327        }
1328        StrategyKind::NgramCache {
1329            proposer,
1330            window_policy,
1331        } => {
1332            validate_window_policy(window_policy, issues);
1333            require_proposer("proposer", proposer, &["ngram-cache"], issues);
1334        }
1335        StrategyKind::NgramSuffix {
1336            proposer,
1337            window_policy,
1338        } => {
1339            validate_window_policy(window_policy, issues);
1340            require_proposer("proposer", proposer, &["ngram-suffix"], issues);
1341        }
1342        StrategyKind::Composite {
1343            primary,
1344            extender,
1345            extension_policy,
1346            window_policy,
1347        } => {
1348            validate_window_policy(window_policy, issues);
1349            if extension_policy.max_tokens == 0 {
1350                push_issue(
1351                    issues,
1352                    ValidationCode::InvalidGenerationValue,
1353                    format!(
1354                        "generation.speculative_decoding.strategies[{name}].extension_policy.max_tokens"
1355                    ),
1356                    "extension_policy.max_tokens must be greater than zero",
1357                );
1358            }
1359            if extension_policy.initial_tokens == Some(0) {
1360                push_issue(
1361                    issues,
1362                    ValidationCode::InvalidGenerationValue,
1363                    format!(
1364                        "generation.speculative_decoding.strategies[{name}].extension_policy.initial_tokens"
1365                    ),
1366                    "extension_policy.initial_tokens must be greater than zero",
1367                );
1368            }
1369            if let Some(initial_tokens) = extension_policy.initial_tokens
1370                && initial_tokens > extension_policy.max_tokens
1371            {
1372                push_issue(
1373                    issues,
1374                    ValidationCode::InvalidGenerationValue,
1375                    format!(
1376                        "generation.speculative_decoding.strategies[{name}].extension_policy.initial_tokens"
1377                    ),
1378                    "extension_policy.initial_tokens must not exceed max_tokens",
1379                );
1380            }
1381            if extension_policy.tail_backoff_proposals == Some(0) {
1382                push_issue(
1383                    issues,
1384                    ValidationCode::InvalidGenerationValue,
1385                    format!(
1386                        "generation.speculative_decoding.strategies[{name}].extension_policy.tail_backoff_proposals"
1387                    ),
1388                    "extension_policy.tail_backoff_proposals must be greater than zero",
1389                );
1390            }
1391            require_proposer("primary", primary, &["native-mtp"], issues);
1392            require_proposer(
1393                "extender",
1394                extender,
1395                &["ngram-cache", "ngram-suffix"],
1396                issues,
1397            );
1398        }
1399    }
1400}
1401
1402#[cfg(test)]
1403mod tests;
skippy_package_format/lib.rs

skippy_package_format/
lib.rs
