1use std::path::Path;
6
7use llama_cpp_4::model::LlamaLoraAdapter;
8use logit_loom::{
9 ControlVectorSpec, Digest, LoraSpec, MAX_TEXT_MECHANICS_LORAS, SteeringAction, SteeringKind,
10 SteeringReceipt,
11};
12
13use crate::{
14 Error, Model, Session,
15 error::native,
16 model::{LORA_ARTIFACT_DOMAIN, digest_file, verify_artifact_length},
17};
18
19#[derive(Clone, Debug, PartialEq, Eq)]
25pub struct AuthorizedLoraArtifact {
26 identity: Digest,
27 byte_length: u64,
28}
29
30impl AuthorizedLoraArtifact {
31 #[must_use]
33 pub const fn new(identity: Digest, byte_length: u64) -> Self {
34 Self {
35 identity,
36 byte_length,
37 }
38 }
39
40 #[must_use]
42 pub const fn identity(&self) -> &Digest {
43 &self.identity
44 }
45
46 #[must_use]
48 pub const fn byte_length(&self) -> u64 {
49 self.byte_length
50 }
51}
52
53pub struct LoraAdapter {
55 native: LlamaLoraAdapter,
56 artifact: Digest,
57}
58
59impl std::fmt::Debug for LoraAdapter {
60 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61 formatter
62 .debug_struct("LoraAdapter")
63 .field("artifact", &self.artifact)
64 .finish_non_exhaustive()
65 }
66}
67
68impl LoraAdapter {
69 pub const fn artifact_digest(&self) -> &Digest {
71 &self.artifact
72 }
73}
74
75pub struct LoraApplication<'adapter> {
77 adapter: &'adapter mut LoraAdapter,
78 scale: f32,
79}
80
81impl<'adapter> LoraApplication<'adapter> {
82 #[must_use]
84 pub const fn new(adapter: &'adapter mut LoraAdapter, scale: f32) -> Self {
85 Self { adapter, scale }
86 }
87
88 #[must_use]
90 pub fn specification(&self) -> LoraSpec {
91 LoraSpec {
92 artifact: self.adapter.artifact.clone(),
93 scale: self.scale,
94 }
95 }
96}
97
98impl Model {
99 pub fn load_lora(&self, path: impl AsRef<Path>) -> Result<LoraAdapter, Error> {
105 let path = path.as_ref();
106 let artifact_before_load = digest_file(path, LORA_ARTIFACT_DOMAIN)?;
107 let native = self.native.lora_adapter_init(path).map_err(native)?;
108 let artifact = digest_file(path, LORA_ARTIFACT_DOMAIN)?;
109 if artifact != artifact_before_load {
110 return Err(Error::Incompatible(
111 "LoRA file changed while llama.cpp was loading it".to_owned(),
112 ));
113 }
114 Ok(LoraAdapter { native, artifact })
115 }
116
117 pub fn load_lora_authorized(
128 &self,
129 path: impl AsRef<Path>,
130 artifact: AuthorizedLoraArtifact,
131 ) -> Result<LoraAdapter, Error> {
132 let path = path.as_ref();
133 verify_artifact_length(path, artifact.byte_length())?;
134 let native = self.native.lora_adapter_init(path).map_err(native)?;
135 verify_artifact_length(path, artifact.byte_length())?;
136 Ok(LoraAdapter {
137 native,
138 artifact: artifact.identity,
139 })
140 }
141}
142
143#[derive(Clone, Debug)]
145pub struct ControlVector {
146 values: Vec<f32>,
147 neutral: Vec<f32>,
148 specification: ControlVectorSpec,
149}
150
151impl ControlVector {
152 pub fn new(
159 values: Vec<f32>,
160 embedding_width: u32,
161 layer_start: i32,
162 layer_end: i32,
163 ) -> Result<Self, Error> {
164 if values.is_empty() || values.iter().any(|value| !value.is_finite()) {
165 return Err(Error::Invalid(
166 "control-vector values must be non-empty and finite".to_owned(),
167 ));
168 }
169 let width = usize::try_from(embedding_width)
170 .map_err(|_| Error::Invalid("control-vector width exceeds usize".to_owned()))?;
171 if width == 0 || !values.len().is_multiple_of(width) {
172 return Err(Error::Invalid(
173 "control-vector length must be divisible by embedding width".to_owned(),
174 ));
175 }
176 let byte_capacity = values
177 .len()
178 .checked_mul(size_of::<f32>())
179 .ok_or_else(|| Error::Invalid("control-vector byte length overflowed".to_owned()))?;
180 let mut bytes = Vec::with_capacity(byte_capacity);
181 for value in &values {
182 bytes.extend_from_slice(&value.to_bits().to_le_bytes());
183 }
184 let specification = ControlVectorSpec {
185 data: Digest::of_bytes("control-vector-f32-le-v1", &bytes),
186 embedding_width,
187 layer_start,
188 layer_end,
189 };
190 specification.validate()?;
191 Ok(Self {
192 neutral: vec![0.0; values.len()],
193 values,
194 specification,
195 })
196 }
197
198 pub const fn specification(&self) -> &ControlVectorSpec {
200 &self.specification
201 }
202}
203
204impl<'model> Session<'model> {
205 pub fn steering_scope<'scope>(
219 &'scope mut self,
220 loras: Vec<LoraApplication<'scope>>,
221 vector: Option<&'scope ControlVector>,
222 ) -> Result<SteeringScope<'scope, 'model>, Error> {
223 self.ensure_steering_available()?;
224 if loras.is_empty() && vector.is_none() {
225 return Err(Error::Invalid(
226 "aggregate steering requires at least one resource".to_owned(),
227 ));
228 }
229 if loras.len() > MAX_TEXT_MECHANICS_LORAS {
230 return Err(Error::Invalid(format!(
231 "aggregate steering exceeds {MAX_TEXT_MECHANICS_LORAS} LoRA applications"
232 )));
233 }
234
235 let position = self.position();
236 let mut active_loras = Vec::with_capacity(loras.len());
237 for selection in loras {
238 let specification = LoraSpec {
239 artifact: selection.adapter.artifact.clone(),
240 scale: selection.scale,
241 };
242 let resource = specification.digest()?;
243 let applied = SteeringReceipt {
244 kind: SteeringKind::Lora,
245 resource: resource.clone(),
246 action: SteeringAction::Applied,
247 position,
248 };
249 applied.digest()?;
250 active_loras.push(ActiveLora {
251 adapter: selection.adapter,
252 scale: selection.scale,
253 resource,
254 applied,
255 });
256 }
257 let active_vector = vector
258 .map(|vector| {
259 vector.validate_for_model(self.model)?;
260 let resource = vector.specification().digest()?;
261 let applied = SteeringReceipt {
262 kind: SteeringKind::ControlVector,
263 resource: resource.clone(),
264 action: SteeringAction::Applied,
265 position,
266 };
267 applied.digest()?;
268 Ok::<ActiveControlVector<'scope>, Error>(ActiveControlVector {
269 vector,
270 resource,
271 applied,
272 })
273 })
274 .transpose()?;
275
276 let mut application_failure = None;
277 for (index, active) in active_loras.iter_mut().enumerate() {
278 if let Err(error) = self
279 .context
280 .lora_adapter_set(&mut active.adapter.native, active.scale)
281 .map_err(native)
282 {
283 application_failure = Some((index, error));
284 break;
285 }
286 }
287 if let Some((applied_loras, error)) = application_failure {
288 return Err(rollback_partial_loras(
289 self,
290 &mut active_loras[..applied_loras],
291 error,
292 ));
293 }
294 if let Some(active) = &active_vector
295 && let Err(error) = apply_control_vector(self, active.vector)
296 {
297 return Err(rollback_partial_steering(
298 self,
299 &mut active_loras,
300 Some(active.vector),
301 error,
302 ));
303 }
304
305 let mut kinds = vec![SteeringKind::Lora; active_loras.len()];
306 if active_vector.is_some() {
307 kinds.push(SteeringKind::ControlVector);
308 }
309 self.mark_steering_stack_active(kinds);
310 Ok(SteeringScope {
311 session: self,
312 loras: active_loras,
313 vector: active_vector,
314 active: true,
315 })
316 }
317
318 pub fn lora_scope<'scope>(
325 &'scope mut self,
326 adapter: &'scope mut LoraAdapter,
327 scale: f32,
328 ) -> Result<LoraScope<'scope, 'model>, Error> {
329 self.ensure_steering_available()?;
330 let specification = LoraSpec {
331 artifact: adapter.artifact.clone(),
332 scale,
333 };
334 let resource = specification.digest()?;
335 let applied = SteeringReceipt {
336 kind: SteeringKind::Lora,
337 resource: resource.clone(),
338 action: SteeringAction::Applied,
339 position: self.position(),
340 };
341 applied.digest()?;
342 self.context
343 .lora_adapter_set(&mut adapter.native, scale)
344 .map_err(native)?;
345 self.mark_steering_active(SteeringKind::Lora);
346 Ok(LoraScope {
347 session: self,
348 adapter,
349 resource,
350 applied,
351 active: true,
352 })
353 }
354
355 pub fn control_vector_scope<'scope>(
362 &'scope mut self,
363 vector: &'scope ControlVector,
364 ) -> Result<ControlVectorScope<'scope, 'model>, Error> {
365 self.ensure_steering_available()?;
366 let specification = vector.specification();
367 vector.validate_for_model(self.model)?;
368 let resource = specification.digest()?;
369 let applied = SteeringReceipt {
370 kind: SteeringKind::ControlVector,
371 resource: resource.clone(),
372 action: SteeringAction::Applied,
373 position: self.position(),
374 };
375 applied.digest()?;
376 self.context
377 .set_adapter_cvec(
378 &vector.values,
379 i32::try_from(specification.embedding_width).map_err(|_| {
380 Error::Invalid("control-vector embedding width exceeds i32".to_owned())
381 })?,
382 specification.layer_start,
383 specification.layer_end,
384 )
385 .map_err(|code| Error::Native(format!("control-vector apply returned {code}")))?;
386 self.mark_steering_active(SteeringKind::ControlVector);
387 Ok(ControlVectorScope {
388 session: self,
389 vector,
390 resource,
391 applied,
392 active: true,
393 })
394 }
395}
396
397struct ActiveLora<'adapter> {
398 adapter: &'adapter mut LoraAdapter,
399 scale: f32,
400 resource: Digest,
401 applied: SteeringReceipt,
402}
403
404struct ActiveControlVector<'vector> {
405 vector: &'vector ControlVector,
406 resource: Digest,
407 applied: SteeringReceipt,
408}
409
410#[must_use = "retain the scope while steering should remain active and call clear to observe cleanup"]
412pub struct SteeringScope<'scope, 'model> {
413 session: &'scope mut Session<'model>,
414 loras: Vec<ActiveLora<'scope>>,
415 vector: Option<ActiveControlVector<'scope>>,
416 active: bool,
417}
418
419impl<'model> SteeringScope<'_, 'model> {
420 pub fn applied_receipts(&self) -> impl Iterator<Item = &SteeringReceipt> {
422 self.loras
423 .iter()
424 .map(|active| &active.applied)
425 .chain(self.vector.iter().map(|active| &active.applied))
426 }
427
428 pub fn session_mut(&mut self) -> &mut Session<'model> {
431 self.session
432 }
433
434 pub fn clear(mut self) -> Result<Vec<SteeringReceipt>, Error> {
442 let result = self.clear_inner();
443 self.active = false;
444 if let Err(error) = &result {
445 self.session.record_cleanup_failure(error);
446 }
447 result
448 }
449
450 fn clear_inner(&mut self) -> Result<Vec<SteeringReceipt>, Error> {
451 let mut cleared = Vec::with_capacity(self.loras.len() + usize::from(self.vector.is_some()));
452 let mut failures = Vec::new();
453 if let Some(active) = &self.vector {
454 match clear_control_vector(self.session, active.vector) {
455 Ok(()) => {
456 let receipt = SteeringReceipt {
457 kind: SteeringKind::ControlVector,
458 resource: active.resource.clone(),
459 action: SteeringAction::Cleared,
460 position: self.session.position(),
461 };
462 receipt.digest()?;
463 cleared.push(receipt);
464 }
465 Err(error) => failures.push(error.to_string()),
466 }
467 }
468 for active in self.loras.iter_mut().rev() {
469 match self
470 .session
471 .context
472 .lora_adapter_remove(&mut active.adapter.native)
473 .map_err(native)
474 {
475 Ok(()) => {
476 let receipt = SteeringReceipt {
477 kind: SteeringKind::Lora,
478 resource: active.resource.clone(),
479 action: SteeringAction::Cleared,
480 position: self.session.position(),
481 };
482 receipt.digest()?;
483 cleared.push(receipt);
484 }
485 Err(error) => failures.push(error.to_string()),
486 }
487 }
488 if !failures.is_empty() {
489 return Err(Error::Poisoned(format!(
490 "aggregate steering cleanup failed: {}",
491 failures.join("; ")
492 )));
493 }
494 self.session.mark_steering_cleared();
495 Ok(cleared)
496 }
497}
498
499impl Drop for SteeringScope<'_, '_> {
500 fn drop(&mut self) {
501 if self.active
502 && let Err(error) = self.clear_inner()
503 {
504 self.session.record_cleanup_failure(&error);
505 }
506 }
507}
508
509#[must_use = "retain the scope while LoRA should remain active and call clear to observe cleanup"]
511pub struct LoraScope<'scope, 'model> {
512 session: &'scope mut Session<'model>,
513 adapter: &'scope mut LoraAdapter,
514 resource: Digest,
515 applied: SteeringReceipt,
516 active: bool,
517}
518
519impl<'model> LoraScope<'_, 'model> {
520 pub const fn applied_receipt(&self) -> &SteeringReceipt {
522 &self.applied
523 }
524
525 pub fn session_mut(&mut self) -> &mut Session<'model> {
527 self.session
528 }
529
530 pub fn clear(mut self) -> Result<SteeringReceipt, Error> {
536 let result = self.clear_inner();
537 self.active = false;
538 if let Err(error) = &result {
539 self.session.record_cleanup_failure(error);
540 }
541 result
542 }
543
544 fn clear_inner(&mut self) -> Result<SteeringReceipt, Error> {
545 let receipt = SteeringReceipt {
546 kind: SteeringKind::Lora,
547 resource: self.resource.clone(),
548 action: SteeringAction::Cleared,
549 position: self.session.position(),
550 };
551 receipt.digest()?;
552 self.session
553 .context
554 .lora_adapter_remove(&mut self.adapter.native)
555 .map_err(native)?;
556 self.session.mark_steering_cleared();
557 Ok(receipt)
558 }
559}
560
561impl Drop for LoraScope<'_, '_> {
562 fn drop(&mut self) {
563 if self.active
564 && let Err(error) = self.clear_inner()
565 {
566 self.session.record_cleanup_failure(&error);
567 }
568 }
569}
570
571#[must_use = "retain the scope while the vector should remain active and call clear to observe cleanup"]
573pub struct ControlVectorScope<'scope, 'model> {
574 session: &'scope mut Session<'model>,
575 vector: &'scope ControlVector,
576 resource: Digest,
577 applied: SteeringReceipt,
578 active: bool,
579}
580
581impl<'model> ControlVectorScope<'_, 'model> {
582 pub const fn applied_receipt(&self) -> &SteeringReceipt {
584 &self.applied
585 }
586
587 pub fn session_mut(&mut self) -> &mut Session<'model> {
589 self.session
590 }
591
592 pub fn clear(mut self) -> Result<SteeringReceipt, Error> {
598 let result = self.clear_inner();
599 self.active = false;
600 if let Err(error) = &result {
601 self.session.record_cleanup_failure(error);
602 }
603 result
604 }
605
606 fn clear_inner(&mut self) -> Result<SteeringReceipt, Error> {
607 let specification = self.vector.specification();
608 let receipt = SteeringReceipt {
609 kind: SteeringKind::ControlVector,
610 resource: self.resource.clone(),
611 action: SteeringAction::Cleared,
612 position: self.session.position(),
613 };
614 receipt.digest()?;
615 self.session
616 .context
617 .set_adapter_cvec(
618 &self.vector.neutral,
619 i32::try_from(specification.embedding_width).map_err(|_| {
620 Error::Invalid("control-vector embedding width exceeds i32".to_owned())
621 })?,
622 specification.layer_start,
623 specification.layer_end,
624 )
625 .map_err(|code| Error::Native(format!("control-vector clear returned {code}")))?;
626 self.session.mark_steering_cleared();
627 Ok(receipt)
628 }
629}
630
631impl Drop for ControlVectorScope<'_, '_> {
632 fn drop(&mut self) {
633 if self.active
634 && let Err(error) = self.clear_inner()
635 {
636 self.session.record_cleanup_failure(&error);
637 }
638 }
639}
640
641impl ControlVector {
642 pub(crate) fn validate_for_model(&self, model: &Model) -> Result<(), Error> {
643 let native_width = u32::try_from(model.native.n_embd())
644 .map_err(|_| Error::Native("model returned an invalid embedding width".to_owned()))?;
645 let native_layers = u32::try_from(model.native.n_layer())
646 .map_err(|_| Error::Native("model returned an invalid layer count".to_owned()))?;
647 if native_layers < 2 || self.specification.embedding_width != native_width {
648 return Err(Error::Incompatible(
649 "control-vector embedding width does not match the model".to_owned(),
650 ));
651 }
652 let last_layer = i32::try_from(native_layers - 1)
653 .map_err(|_| Error::Native("model layer count exceeds i32".to_owned()))?;
654 if self.specification.layer_end > last_layer {
655 return Err(Error::Incompatible(format!(
656 "control-vector layer end exceeds model layer {last_layer}"
657 )));
658 }
659 let width = usize::try_from(native_width)
660 .map_err(|_| Error::Native("model embedding width exceeds usize".to_owned()))?;
661 let rows = usize::try_from(native_layers - 1)
662 .map_err(|_| Error::Native("model layer count exceeds usize".to_owned()))?;
663 let expected = width
664 .checked_mul(rows)
665 .ok_or_else(|| Error::Invalid("control-vector dimensions overflowed".to_owned()))?;
666 if self.values.len() != expected {
667 return Err(Error::Incompatible(format!(
668 "control vector contains {} values; model requires {expected}",
669 self.values.len()
670 )));
671 }
672 Ok(())
673 }
674}
675
676pub(crate) fn validate_steering_resources(
677 model: &Model,
678 loras: &[LoraApplication<'_>],
679 vector: Option<&ControlVector>,
680) -> Result<Vec<LoraSpec>, Error> {
681 if loras.len() > MAX_TEXT_MECHANICS_LORAS {
682 return Err(Error::Invalid(format!(
683 "aggregate steering exceeds {MAX_TEXT_MECHANICS_LORAS} LoRA applications"
684 )));
685 }
686 let specifications = loras
687 .iter()
688 .map(LoraApplication::specification)
689 .collect::<Vec<_>>();
690 for specification in &specifications {
691 specification.validate()?;
692 }
693 if let Some(vector) = vector {
694 vector.specification().validate()?;
695 vector.validate_for_model(model)?;
696 }
697 Ok(specifications)
698}
699
700pub(crate) fn execute_with_steering<'model, T>(
701 session: &mut Session<'model>,
702 loras: Vec<LoraApplication<'_>>,
703 vector: Option<&ControlVector>,
704 operation: impl FnOnce(&mut Session<'model>) -> Result<T, Error>,
705) -> Result<(T, Vec<SteeringReceipt>, Vec<SteeringReceipt>), Error> {
706 if loras.is_empty() && vector.is_none() {
707 return operation(session).map(|output| (output, Vec::new(), Vec::new()));
708 }
709
710 let mut scope = session.steering_scope(loras, vector)?;
711 let applied = scope.applied_receipts().cloned().collect::<Vec<_>>();
712 let operation_result = operation(scope.session_mut());
713 let cleanup_result = scope.clear();
714 match (operation_result, cleanup_result) {
715 (Ok(output), Ok(cleared)) => Ok((output, applied, cleared)),
716 (Err(operation_error), Ok(_)) => Err(operation_error),
717 (Ok(_), Err(cleanup_error)) => Err(cleanup_error),
718 (Err(operation_error), Err(cleanup_error)) => Err(Error::Poisoned(format!(
719 "text operation failed ({operation_error}); steering cleanup also failed ({cleanup_error})"
720 ))),
721 }
722}
723
724fn apply_control_vector(session: &mut Session<'_>, vector: &ControlVector) -> Result<(), Error> {
725 let specification = vector.specification();
726 session
727 .context
728 .set_adapter_cvec(
729 &vector.values,
730 i32::try_from(specification.embedding_width).map_err(|_| {
731 Error::Invalid("control-vector embedding width exceeds i32".to_owned())
732 })?,
733 specification.layer_start,
734 specification.layer_end,
735 )
736 .map_err(|code| Error::Native(format!("control-vector apply returned {code}")))
737}
738
739fn clear_control_vector(session: &mut Session<'_>, vector: &ControlVector) -> Result<(), Error> {
740 let specification = vector.specification();
741 session
742 .context
743 .set_adapter_cvec(
744 &vector.neutral,
745 i32::try_from(specification.embedding_width).map_err(|_| {
746 Error::Invalid("control-vector embedding width exceeds i32".to_owned())
747 })?,
748 specification.layer_start,
749 specification.layer_end,
750 )
751 .map_err(|code| Error::Native(format!("control-vector clear returned {code}")))
752}
753
754fn rollback_partial_loras(
755 session: &mut Session<'_>,
756 loras: &mut [ActiveLora<'_>],
757 application_error: Error,
758) -> Error {
759 let mut failures = Vec::new();
760 for active in loras.iter_mut().rev() {
761 if let Err(cleanup_error) = session
762 .context
763 .lora_adapter_remove(&mut active.adapter.native)
764 .map_err(native)
765 {
766 failures.push(cleanup_error.to_string());
767 }
768 }
769 if failures.is_empty() {
770 application_error
771 } else {
772 let error = Error::Poisoned(format!(
773 "steering application failed ({application_error}); partial LoRA rollback failed: {}",
774 failures.join("; ")
775 ));
776 session.record_cleanup_failure(&error);
777 error
778 }
779}
780
781fn rollback_partial_steering(
782 session: &mut Session<'_>,
783 loras: &mut [ActiveLora<'_>],
784 vector: Option<&ControlVector>,
785 application_error: Error,
786) -> Error {
787 let mut failures = Vec::new();
788 if let Some(vector) = vector
789 && let Err(cleanup_error) = clear_control_vector(session, vector)
790 {
791 failures.push(format!("control vector: {cleanup_error}"));
792 }
793 for active in loras.iter_mut().rev() {
794 if let Err(cleanup_error) = session
795 .context
796 .lora_adapter_remove(&mut active.adapter.native)
797 .map_err(native)
798 {
799 failures.push(format!("LoRA: {cleanup_error}"));
800 }
801 }
802 if failures.is_empty() {
803 application_error
804 } else {
805 let error = Error::Poisoned(format!(
806 "steering application failed ({application_error}); aggregate rollback failed: {}",
807 failures.join("; ")
808 ));
809 session.record_cleanup_failure(&error);
810 error
811 }
812}
813
814#[cfg(test)]
815mod tests {
816 use super::*;
817
818 #[test]
819 fn control_vector_builds_a_matching_neutral_buffer() {
820 let vector = ControlVector::new(vec![1.0, -1.0, 0.5, 0.0], 2, 1, 1).unwrap();
821 assert_eq!(vector.values.len(), vector.neutral.len());
822 assert!(vector.neutral.iter().all(|value| *value == 0.0));
823 assert_eq!(vector.specification.layer_end, 1);
824 }
825
826 #[test]
827 fn control_vector_rejects_malformed_values_and_ranges() {
828 assert!(ControlVector::new(vec![f32::NAN, 0.0], 2, 1, 1).is_err());
829 assert!(ControlVector::new(vec![0.0, 1.0, 2.0], 2, 1, 1).is_err());
830 assert!(ControlVector::new(vec![0.0, 1.0], 2, 0, 1).is_err());
831 }
832
833 #[test]
834 fn authorized_lora_preserves_provider_identity_without_payload() {
835 let identity = Digest::of_bytes("provider-artifact-id-v1", b"text-lora-one");
836 let artifact = AuthorizedLoraArtifact::new(identity.clone(), 4_194_304);
837 assert_eq!(artifact.identity(), &identity);
838 assert_eq!(artifact.byte_length(), 4_194_304);
839 }
840}