use serde::{Deserialize, Serialize};
use crate::{
CandidateMode, CoreError, Digest, GenerationFinish, GenerationPlanV2, GenerationReceipt,
TokenId,
};
pub const MAX_STRUCTURED_SCOPE_BYTES: usize = 256;
pub const MAX_STRUCTURED_ATTEMPTS: u32 = 64;
pub const MAX_STRUCTURED_OUTPUT_BYTES: u32 = 1024 * 1024;
pub const MAX_STRUCTURED_STEPS: u32 = 65_536;
pub const MAX_STRUCTURED_CANDIDATES: u32 = 1_000_000;
const OUTPUT_DOMAIN: &str = "structured-projection-output-v1";
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredMechanicIdentityV1 {
pub implementation: Digest,
pub configuration: Digest,
}
impl StructuredMechanicIdentityV1 {
pub fn digest(&self) -> Result<Digest, CoreError> {
Digest::of_serializable("structured-mechanic-identity-v1", self)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ByteFeedbackSpecV1 {
pub mechanic: StructuredMechanicIdentityV1,
pub mode: CandidateMode,
pub max_steps: u32,
pub max_output_bytes: u32,
}
impl ByteFeedbackSpecV1 {
pub fn validate(&self) -> Result<(), CoreError> {
self.mode.validate()?;
if !(1..=MAX_STRUCTURED_STEPS).contains(&self.max_steps)
|| !(1..=MAX_STRUCTURED_OUTPUT_BYTES).contains(&self.max_output_bytes)
{
return Err(CoreError::invalid(
"structured byte feedback",
"step and output-byte bounds must be positive and within public maxima",
));
}
Ok(())
}
pub fn digest(&self) -> Result<Digest, CoreError> {
self.validate()?;
Digest::of_serializable("byte-feedback-spec-v1", self)
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredProjectionPlanV1 {
pub scope: String,
pub target_topology: Digest,
pub constraints: Digest,
pub compiler: StructuredMechanicIdentityV1,
pub validator: StructuredMechanicIdentityV1,
pub feedback: ByteFeedbackSpecV1,
pub generation: GenerationPlanV2,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub transform_pipeline: Option<Digest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub observer_set: Option<Digest>,
pub maximum_attempts: u32,
}
impl StructuredProjectionPlanV1 {
pub fn validate(&self) -> Result<(), CoreError> {
if self.scope.trim().is_empty()
|| self.scope.len() > MAX_STRUCTURED_SCOPE_BYTES
|| self.scope.contains('\0')
{
return Err(CoreError::invalid(
"structured projection scope",
format!(
"must be non-empty, NUL-free, and at most {MAX_STRUCTURED_SCOPE_BYTES} bytes"
),
));
}
self.compiler.digest()?;
self.validator.digest()?;
self.feedback.validate()?;
self.generation.validate()?;
if self.generation.grammar.is_none() {
return Err(CoreError::invalid(
"structured projection grammar",
"one explicit eager or lazy grammar is required",
));
}
if self.generation.generation.max_tokens > self.feedback.max_steps {
return Err(CoreError::invalid(
"structured projection generation",
"generation token bound exceeds byte-feedback step capacity",
));
}
if !(1..=MAX_STRUCTURED_ATTEMPTS).contains(&self.maximum_attempts) {
return Err(CoreError::invalid(
"structured projection attempts",
format!("must be in 1..={MAX_STRUCTURED_ATTEMPTS}"),
));
}
Ok(())
}
pub fn digest(&self) -> Result<Digest, CoreError> {
self.validate()?;
Digest::of_serializable("structured-projection-plan-v1", self)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredCompilationReceiptV1 {
pub constraints: Digest,
pub compiler: Digest,
pub grammar: Digest,
pub feedback: Digest,
pub plan: Digest,
}
impl StructuredCompilationReceiptV1 {
pub fn validate_for(&self, plan: &StructuredProjectionPlanV1) -> Result<(), CoreError> {
plan.validate()?;
let grammar = plan.generation.grammar.as_ref().ok_or_else(|| {
CoreError::invalid("structured compilation", "compiled grammar is absent")
})?;
if self.constraints != plan.constraints
|| self.compiler != plan.compiler.digest()?
|| self.grammar != grammar.digest()?
|| self.feedback != plan.feedback.digest()?
|| self.plan != plan.digest()?
{
return Err(CoreError::invalid(
"structured compilation receipt",
"compiler input or output identity differs from the plan",
));
}
Ok(())
}
pub fn digest_for(&self, plan: &StructuredProjectionPlanV1) -> Result<Digest, CoreError> {
self.validate_for(plan)?;
Digest::of_serializable("structured-compilation-receipt-v1", self)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ByteFeedbackStepReceiptV1 {
pub step: u32,
pub accepted_tokens: u32,
pub output_bytes: u32,
pub output_prefix: Digest,
pub state: Digest,
pub candidate_tokens: u32,
pub eligible_candidates: u32,
pub allowed_candidates: u32,
pub rejected_candidates: u32,
pub allowed_tokens: Digest,
pub rejected_tokens: Digest,
}
impl ByteFeedbackStepReceiptV1 {
fn validate_for(&self, specification: &ByteFeedbackSpecV1) -> Result<(), CoreError> {
if self.step >= specification.max_steps
|| self.accepted_tokens > specification.max_steps
|| self.output_bytes > specification.max_output_bytes
|| self.candidate_tokens == 0
|| self.eligible_candidates > self.candidate_tokens
|| self.allowed_candidates == 0
|| self.allowed_candidates > self.eligible_candidates
|| self
.allowed_candidates
.checked_add(self.rejected_candidates)
!= Some(self.eligible_candidates)
{
return Err(CoreError::invalid(
"byte-feedback step",
"step, output, or candidate accounting is inconsistent",
));
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ByteFeedbackReceiptV1 {
pub feedback: Digest,
pub tokenizer: Digest,
pub initial_tokens: Digest,
pub initial_token_count: u64,
pub output: Digest,
pub output_bytes: u32,
pub accepted_tokens: u32,
pub steps: Vec<ByteFeedbackStepReceiptV1>,
pub final_state: Digest,
pub complete: bool,
}
impl ByteFeedbackReceiptV1 {
pub fn validate_for(&self, specification: &ByteFeedbackSpecV1) -> Result<(), CoreError> {
specification.validate()?;
if self.feedback != specification.digest()?
|| self.output_bytes > specification.max_output_bytes
|| self.accepted_tokens > specification.max_steps
|| self.steps.len() > usize::try_from(specification.max_steps).unwrap_or(usize::MAX)
{
return Err(CoreError::invalid(
"byte-feedback receipt",
"specification, output, or step accounting differs",
));
}
let mut prior_tokens = 0;
let mut prior_bytes = 0;
for (expected, step) in (0_u32..).zip(&self.steps) {
step.validate_for(specification)?;
if step.step != expected
|| step.accepted_tokens < prior_tokens
|| step.output_bytes < prior_bytes
|| step.accepted_tokens > self.accepted_tokens
|| step.output_bytes > self.output_bytes
{
return Err(CoreError::invalid(
"byte-feedback step order",
"steps or output prefixes are not contiguous and monotonic",
));
}
prior_tokens = step.accepted_tokens;
prior_bytes = step.output_bytes;
}
Ok(())
}
pub fn digest_for(&self, specification: &ByteFeedbackSpecV1) -> Result<Digest, CoreError> {
self.validate_for(specification)?;
Digest::of_serializable("byte-feedback-receipt-v1", self)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StructuredValidationStatusV1 {
Conforms,
Rejected,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredValidationReceiptV1 {
pub validator: Digest,
pub constraints: Digest,
pub source: Digest,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub canonical_artifact: Option<Digest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub validation_report: Option<Digest>,
pub status: StructuredValidationStatusV1,
pub violations: u32,
pub diagnostics: u32,
}
impl StructuredValidationReceiptV1 {
pub fn validate_for(
&self,
plan: &StructuredProjectionPlanV1,
output: &Digest,
) -> Result<(), CoreError> {
let valid_status = match self.status {
StructuredValidationStatusV1::Conforms => self.violations == 0 && self.diagnostics == 0,
StructuredValidationStatusV1::Rejected => self.violations > 0 || self.diagnostics > 0,
};
if self.validator != plan.validator.digest()?
|| self.constraints != plan.constraints
|| self.source != *output
|| !valid_status
{
return Err(CoreError::invalid(
"structured validation receipt",
"validator, source, constraints, or disposition accounting differs",
));
}
Ok(())
}
pub fn digest_for(
&self,
plan: &StructuredProjectionPlanV1,
output: &Digest,
) -> Result<Digest, CoreError> {
self.validate_for(plan, output)?;
Digest::of_serializable("structured-validation-receipt-v1", self)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StructuredAttemptDispositionV1 {
Cancelled,
Rejected,
Selected,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredAttemptReceiptV1 {
pub attempt: u32,
pub output: Digest,
pub generation: GenerationReceipt,
pub feedback: ByteFeedbackReceiptV1,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub validation: Option<StructuredValidationReceiptV1>,
pub disposition: StructuredAttemptDispositionV1,
pub boundary_state: Digest,
pub terminal_state: Digest,
}
impl StructuredAttemptReceiptV1 {
pub fn digest_for(&self, plan: &StructuredProjectionPlanV1) -> Result<Digest, CoreError> {
self.generation.digest()?;
self.feedback.validate_for(&plan.feedback)?;
if self.generation.plan != plan.generation.digest()?
|| self.output != self.feedback.output
|| u64::from(self.feedback.output_bytes) != self.generation.admitted_bytes
|| self.feedback.accepted_tokens != self.generation.admitted_tokens
|| self.generation.transform_receipt.is_some() != plan.transform_pipeline.is_some()
|| self.generation.observer_receipts.is_empty() == plan.observer_set.is_some()
{
return Err(CoreError::invalid(
"structured projection attempt",
"generation, output, or feedback accounting differs from the plan",
));
}
if let Some(validation) = &self.validation {
validation.validate_for(plan, &self.output)?;
}
match self.disposition {
StructuredAttemptDispositionV1::Cancelled => {
if !matches!(self.generation.finish, GenerationFinish::ObserverStop)
|| self.validation.is_some()
|| self.terminal_state != self.boundary_state
{
return Err(CoreError::invalid(
"cancelled structured attempt",
"cancellation must be unvalidated and restore the boundary",
));
}
}
StructuredAttemptDispositionV1::Rejected => {
if matches!(self.generation.finish, GenerationFinish::ObserverStop)
|| self.validation.as_ref().is_none_or(|validation| {
validation.status != StructuredValidationStatusV1::Rejected
})
|| self.terminal_state != self.boundary_state
{
return Err(CoreError::invalid(
"rejected structured attempt",
"rejection requires authoritative nonconformance and boundary restore",
));
}
}
StructuredAttemptDispositionV1::Selected => {
if matches!(self.generation.finish, GenerationFinish::ObserverStop)
|| self.validation.as_ref().is_none_or(|validation| {
validation.status != StructuredValidationStatusV1::Conforms
})
|| self.terminal_state == self.boundary_state
|| !self.feedback.complete
{
return Err(CoreError::invalid(
"selected structured attempt",
"selection requires complete conforming output retained in causal state",
));
}
}
}
Digest::of_serializable("structured-attempt-receipt-v1", self)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StructuredProjectionReceiptV1 {
pub plan: Digest,
pub compilation: Digest,
pub boundary_state: Digest,
pub attempts: Vec<StructuredAttemptReceiptV1>,
pub selected_attempt: u32,
pub final_state: Digest,
}
impl StructuredProjectionReceiptV1 {
pub fn validate_for(
&self,
plan: &StructuredProjectionPlanV1,
compilation: &StructuredCompilationReceiptV1,
) -> Result<(), CoreError> {
if self.plan != plan.digest()?
|| self.compilation != compilation.digest_for(plan)?
|| self.attempts.is_empty()
|| self.attempts.len() > usize::try_from(plan.maximum_attempts).unwrap_or(usize::MAX)
{
return Err(CoreError::invalid(
"structured projection receipt",
"plan, compilation, or attempt bounds differ",
));
}
let mut selected = None;
for (expected, attempt) in (0_u32..).zip(&self.attempts) {
attempt.digest_for(plan)?;
if attempt.attempt != expected || attempt.boundary_state != self.boundary_state {
return Err(CoreError::invalid(
"structured projection attempt lineage",
"attempt order or common boundary differs",
));
}
if attempt.disposition == StructuredAttemptDispositionV1::Selected
&& selected.replace(attempt).is_some()
{
return Err(CoreError::invalid(
"structured projection selection",
"more than one attempt was selected",
));
}
}
let Some(selected) = selected else {
return Err(CoreError::invalid(
"structured projection selection",
"one selected terminal attempt is required",
));
};
if selected.attempt != self.selected_attempt
|| selected.attempt.checked_add(1) != u32::try_from(self.attempts.len()).ok()
|| selected.terminal_state != self.final_state
{
return Err(CoreError::invalid(
"structured projection selection",
"the selected attempt must be last and bind the final state",
));
}
Ok(())
}
pub fn digest_for(
&self,
plan: &StructuredProjectionPlanV1,
compilation: &StructuredCompilationReceiptV1,
) -> Result<Digest, CoreError> {
self.validate_for(plan, compilation)?;
Digest::of_serializable("structured-projection-receipt-v1", self)
}
}
#[must_use]
pub fn structured_output_digest(bytes: &[u8]) -> Digest {
Digest::of_bytes(OUTPUT_DOMAIN, bytes)
}
pub fn structured_tokens_digest(tokens: &[TokenId]) -> Result<Digest, CoreError> {
Digest::of_serializable("structured-projection-tokens-v1", tokens)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
GenerationPlan, Grammar, GrammarActivationV2, GrammarPlanV2, SamplingPlan,
StructuredValidationStatusV1,
};
fn digest(label: &str) -> Digest {
Digest::of_bytes("structured-contract-test-v1", label.as_bytes())
}
fn mechanic(label: &str) -> StructuredMechanicIdentityV1 {
StructuredMechanicIdentityV1 {
implementation: digest(label),
configuration: digest(&format!("{label}-configuration")),
}
}
fn plan() -> StructuredProjectionPlanV1 {
StructuredProjectionPlanV1 {
scope: "mechanical-json".to_owned(),
target_topology: digest("target-topology"),
constraints: digest("constraints"),
compiler: mechanic("compiler"),
validator: mechanic("validator"),
feedback: ByteFeedbackSpecV1 {
mechanic: mechanic("feedback"),
mode: CandidateMode::FullVocabulary,
max_steps: 4,
max_output_bytes: 128,
},
generation: GenerationPlanV2 {
generation: GenerationPlan {
sampling: SamplingPlan::default(),
max_tokens: 4,
biases: Vec::new(),
grammar: None,
stops: Vec::new(),
},
grammar: Some(GrammarPlanV2 {
grammar: Grammar {
source: "root ::= \"{}\"".to_owned(),
root: "root".to_owned(),
},
activation: GrammarActivationV2::Eager,
}),
},
transform_pipeline: None,
observer_set: None,
maximum_attempts: 2,
}
}
fn compilation(plan: &StructuredProjectionPlanV1) -> StructuredCompilationReceiptV1 {
StructuredCompilationReceiptV1 {
constraints: plan.constraints.clone(),
compiler: plan.compiler.digest().unwrap(),
grammar: plan.generation.grammar.as_ref().unwrap().digest().unwrap(),
feedback: plan.feedback.digest().unwrap(),
plan: plan.digest().unwrap(),
}
}
fn feedback(plan: &StructuredProjectionPlanV1) -> ByteFeedbackReceiptV1 {
ByteFeedbackReceiptV1 {
feedback: plan.feedback.digest().unwrap(),
tokenizer: digest("tokenizer"),
initial_tokens: digest("initial-tokens"),
initial_token_count: 3,
output: structured_output_digest(b"{}"),
output_bytes: 2,
accepted_tokens: 1,
steps: vec![ByteFeedbackStepReceiptV1 {
step: 0,
accepted_tokens: 0,
output_bytes: 0,
output_prefix: structured_output_digest(b""),
state: digest("state"),
candidate_tokens: 16,
eligible_candidates: 8,
allowed_candidates: 2,
rejected_candidates: 6,
allowed_tokens: digest("allowed"),
rejected_tokens: digest("rejected"),
}],
final_state: digest("final-feedback"),
complete: true,
}
}
fn selected_attempt(plan: &StructuredProjectionPlanV1) -> StructuredAttemptReceiptV1 {
let feedback = feedback(plan);
StructuredAttemptReceiptV1 {
attempt: 0,
output: feedback.output.clone(),
generation: GenerationReceipt {
plan: plan.generation.digest().unwrap(),
initial_position: 3,
admitted_tokens: 1,
admitted_bytes: 2,
final_position: 4,
finish: GenerationFinish::TokenLimit,
transform_receipt: None,
observer_receipts: Vec::new(),
},
feedback,
validation: Some(StructuredValidationReceiptV1 {
validator: plan.validator.digest().unwrap(),
constraints: plan.constraints.clone(),
source: structured_output_digest(b"{}"),
canonical_artifact: Some(digest("canonical")),
validation_report: None,
status: StructuredValidationStatusV1::Conforms,
violations: 0,
diagnostics: 0,
}),
disposition: StructuredAttemptDispositionV1::Selected,
boundary_state: digest("boundary"),
terminal_state: digest("selected"),
}
}
#[test]
fn compiled_plan_binds_grammar_feedback_and_attempt_bounds() {
let valid = plan();
valid.validate().unwrap();
compilation(&valid).validate_for(&valid).unwrap();
let mut no_grammar = valid.clone();
no_grammar.generation.grammar = None;
assert!(no_grammar.validate().is_err());
let mut excessive_generation = valid;
excessive_generation.generation.generation.max_tokens = 5;
assert!(excessive_generation.validate().is_err());
}
#[test]
fn projection_requires_one_final_selected_attempt() {
let plan = plan();
let compilation = compilation(&plan);
let selected = selected_attempt(&plan);
let receipt = StructuredProjectionReceiptV1 {
plan: plan.digest().unwrap(),
compilation: compilation.digest_for(&plan).unwrap(),
boundary_state: digest("boundary"),
attempts: vec![selected.clone()],
selected_attempt: 0,
final_state: digest("selected"),
};
receipt.validate_for(&plan, &compilation).unwrap();
let mut not_terminal = receipt;
let mut rejected = selected;
rejected.attempt = 1;
rejected.disposition = StructuredAttemptDispositionV1::Rejected;
rejected.terminal_state = digest("boundary");
rejected.validation.as_mut().unwrap().status = StructuredValidationStatusV1::Rejected;
rejected.validation.as_mut().unwrap().violations = 1;
not_terminal.attempts.push(rejected);
assert!(not_terminal.validate_for(&plan, &compilation).is_err());
}
}