1mod convolution;
14mod crt;
15mod encoding;
16mod layout_compat;
17mod mat_znx;
18mod module;
19mod plan_cache;
20mod scalar_znx;
21mod scratch;
22mod scratch_views;
23mod serialization;
24mod stats;
25mod svp_ppol;
26mod vec_znx;
27mod vec_znx_big;
28mod vec_znx_dft;
29mod vmp_pmat;
30mod word;
31mod znx_base;
32
33pub use convolution::*;
34pub use crt::*;
35pub use layout_compat::*;
36pub use mat_znx::*;
37pub use module::*;
38pub use plan_cache::*;
39pub use scalar_znx::*;
40pub use scratch::*;
41pub use scratch_views::*;
42pub use serialization::*;
43pub use stats::*;
44pub use svp_ppol::*;
45pub use vec_znx::*;
46pub use vec_znx_big::*;
47pub use vec_znx_dft::*;
48pub use vmp_pmat::*;
49pub use word::*;
50pub use znx_base::*;
51
52use anyhow::Result;
53use std::ptr::NonNull;
54
55use crate::oep::HalModuleImpl;
56
57pub trait Data = PartialEq + Eq + Sized + Default;
63
64pub trait HostDataRef = Data + AsRef<[u8]> + Sync;
70
71pub trait HostDataMut = HostDataRef + AsMut<[u8]> + Send;
77
78#[inline]
79pub(crate) fn checked_product(factors: &[usize], context: &str) -> usize {
80 factors
81 .iter()
82 .copied()
83 .try_fold(1usize, usize::checked_mul)
84 .unwrap_or_else(|| {
85 panic!("{context} overflows usize");
86 })
87}
88
89mod private {
90 pub trait Sealed {}
91}
92
93pub trait Location: private::Sealed {}
100
101#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
103pub struct Host;
104
105#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
107pub struct Device;
108
109impl private::Sealed for Host {}
110impl private::Sealed for Device {}
111impl Location for Host {}
112impl Location for Device {}
113
114pub trait HostBackend: Backend<Location = Host> {}
116impl<BE: Backend<Location = Host>> HostBackend for BE {}
117
118pub trait HostVisibleBackend: HostBackend
120where
121 for<'a> Self::BufRef<'a>: AsRef<[u8]>,
122 for<'a> Self::BufMut<'a>: AsRef<[u8]> + AsMut<[u8]>,
123{
124}
125
126impl<BE> HostVisibleBackend for BE
127where
128 BE: HostBackend,
129 for<'a> BE::BufRef<'a>: AsRef<[u8]>,
130 for<'a> BE::BufMut<'a>: AsRef<[u8]> + AsMut<[u8]>,
131{
132}
133
134#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
137pub struct HostBytesBackend;
138
139impl Backend for HostBytesBackend {
140 type TaskExecutor = crate::execution::SerialTaskExecutor;
141 type ZnxWord = i64;
142 type BigWord = i128;
143 type DftWord = i64;
144 type OwnedBuf = Vec<u8>;
145 type BufRef<'a> = &'a [u8];
146 type BufMut<'a> = &'a mut [u8];
147 type Handle = ();
148 type Location = Host;
149
150 fn alloc_bytes(len: usize) -> Self::OwnedBuf {
151 crate::alloc_aligned::<u8>(len)
152 }
153
154 fn alloc_zeroed_bytes(len: usize) -> Self::OwnedBuf {
155 crate::alloc_aligned::<u8>(len)
156 }
157
158 fn from_host_bytes(bytes: &[u8]) -> Self::OwnedBuf {
159 let mut out = crate::alloc_aligned::<u8>(bytes.len());
160 out.copy_from_slice(bytes);
161 out
162 }
163
164 fn from_bytes(bytes: Vec<u8>) -> Self::OwnedBuf {
165 if crate::is_aligned(bytes.as_ptr()) {
166 bytes
167 } else {
168 let mut out = crate::alloc_aligned::<u8>(bytes.len());
169 out.copy_from_slice(&bytes);
170 out
171 }
172 }
173
174 fn to_host_bytes(buf: &Self::OwnedBuf) -> Vec<u8> {
175 buf.clone()
176 }
177
178 fn copy_to_host(buf: &Self::OwnedBuf, dst: &mut [u8]) {
179 assert!(
180 buf.len() >= dst.len(),
181 "backend buffer length {} is smaller than destination host slice length {}",
182 buf.len(),
183 dst.len()
184 );
185 dst.copy_from_slice(&buf[..dst.len()]);
186 }
187
188 fn copy_from_host(buf: &mut Self::OwnedBuf, src: &[u8]) {
189 assert!(
190 buf.len() >= src.len(),
191 "backend buffer length {} is smaller than source host slice length {}",
192 buf.len(),
193 src.len()
194 );
195 let src_len = src.len();
196 buf[..src_len].copy_from_slice(src);
197 buf[src_len..].fill(0);
198 }
199
200 fn copy_view_to_host(buf: &Self::BufRef<'_>, dst: &mut [u8]) {
201 assert_eq!(buf.len(), dst.len());
202 dst.copy_from_slice(buf);
203 }
204
205 fn copy_host_to_view(buf: &mut Self::BufMut<'_>, src: &[u8]) {
206 assert_eq!(buf.len(), src.len());
207 buf.copy_from_slice(src);
208 }
209
210 fn len_bytes(buf: &Self::OwnedBuf) -> usize {
211 buf.len()
212 }
213
214 fn len_bytes_ref(buf: &Self::BufRef<'_>) -> usize {
215 buf.len()
216 }
217
218 fn len_bytes_mut(buf: &Self::BufMut<'_>) -> usize {
219 buf.len()
220 }
221
222 fn view(buf: &Self::OwnedBuf) -> Self::BufRef<'_> {
223 buf.as_slice()
224 }
225
226 fn view_ref<'a, 'b>(buf: &'a Self::BufRef<'b>) -> Self::BufRef<'a>
227 where
228 Self: 'b,
229 {
230 buf
231 }
232
233 fn view_ref_mut<'a, 'b>(buf: &'a Self::BufMut<'b>) -> Self::BufRef<'a>
234 where
235 Self: 'b,
236 {
237 buf
238 }
239
240 fn view_mut_ref<'a, 'b>(buf: &'a mut Self::BufMut<'b>) -> Self::BufMut<'a>
241 where
242 Self: 'b,
243 {
244 buf
245 }
246
247 fn view_mut(buf: &mut Self::OwnedBuf) -> Self::BufMut<'_> {
248 buf.as_mut_slice()
249 }
250
251 fn region(buf: &Self::OwnedBuf, offset: usize, len: usize) -> Self::BufRef<'_> {
252 &buf[offset..offset + len]
253 }
254
255 fn region_mut(buf: &mut Self::OwnedBuf, offset: usize, len: usize) -> Self::BufMut<'_> {
256 &mut buf[offset..offset + len]
257 }
258
259 fn region_ref<'a, 'b>(buf: &'a Self::BufRef<'b>, offset: usize, len: usize) -> Self::BufRef<'a>
260 where
261 Self: 'b,
262 {
263 &buf[offset..offset + len]
264 }
265
266 fn region_ref_mut<'a, 'b>(buf: &'a Self::BufMut<'b>, offset: usize, len: usize) -> Self::BufRef<'a>
267 where
268 Self: 'b,
269 {
270 &buf[offset..offset + len]
271 }
272
273 fn region_mut_ref<'a, 'b>(buf: &'a mut Self::BufMut<'b>, offset: usize, len: usize) -> Self::BufMut<'a>
274 where
275 Self: 'b,
276 {
277 &mut buf[offset..offset + len]
278 }
279
280 unsafe fn destroy(_handle: NonNull<Self::Handle>) {}
281}
282
283unsafe impl HalModuleImpl<HostBytesBackend> for HostBytesBackend {
284 fn new(n: u64) -> crate::layouts::Module<Self> {
285 assert!(n.is_power_of_two(), "n must be a power of two, got {n}");
286 unsafe { crate::layouts::Module::from_nonnull(NonNull::dangling(), n) }
287 }
288}
289
290pub trait DeviceBackend: Backend<Location = Device> {}
292impl<BE: Backend<Location = Device>> DeviceBackend for BE {}
293
294pub trait ToOwnedDeep {
301 type Owned;
302 fn to_owned_deep(&self) -> Self::Owned;
303}
304
305pub trait DigestU64 {
312 fn digest_u64(&self) -> u64;
313}
314
315pub type OwnedBuf<BE> = <BE as Backend>::OwnedBuf;
317
318pub trait CopyToHost {
325 fn len_bytes(&self) -> usize;
327
328 fn copy_to_host(&self, dst: &mut [u8]);
330
331 fn as_host_bytes(&self) -> Option<&[u8]> {
334 None
335 }
336}
337
338pub trait CopyFromHost {
340 fn len_bytes(&self) -> usize;
342
343 fn copy_from_host(&mut self, src: &[u8]);
345
346 fn as_host_bytes_mut(&mut self) -> Option<&mut [u8]> {
348 None
349 }
350}
351
352impl CopyToHost for Vec<u8> {
353 fn len_bytes(&self) -> usize {
354 self.len()
355 }
356 fn copy_to_host(&self, dst: &mut [u8]) {
357 dst.copy_from_slice(self);
358 }
359 fn as_host_bytes(&self) -> Option<&[u8]> {
360 Some(self)
361 }
362}
363
364impl CopyFromHost for Vec<u8> {
365 fn len_bytes(&self) -> usize {
366 self.len()
367 }
368 fn copy_from_host(&mut self, src: &[u8]) {
369 self.copy_from_slice(src);
370 }
371 fn as_host_bytes_mut(&mut self) -> Option<&mut [u8]> {
372 Some(self)
373 }
374}
375
376impl CopyToHost for &[u8] {
377 fn len_bytes(&self) -> usize {
378 <[u8]>::len(self)
379 }
380 fn copy_to_host(&self, dst: &mut [u8]) {
381 dst.copy_from_slice(self);
382 }
383 fn as_host_bytes(&self) -> Option<&[u8]> {
384 Some(self)
385 }
386}
387
388impl CopyToHost for &mut [u8] {
389 fn len_bytes(&self) -> usize {
390 <[u8]>::len(self)
391 }
392 fn copy_to_host(&self, dst: &mut [u8]) {
393 dst.copy_from_slice(self);
394 }
395 fn as_host_bytes(&self) -> Option<&[u8]> {
396 Some(self)
397 }
398}
399
400impl CopyFromHost for &mut [u8] {
401 fn len_bytes(&self) -> usize {
402 <[u8]>::len(self)
403 }
404 fn copy_from_host(&mut self, src: &[u8]) {
405 self.copy_from_slice(src);
406 }
407 fn as_host_bytes_mut(&mut self) -> Option<&mut [u8]> {
408 Some(self)
409 }
410}
411
412pub fn transfer_buf_into<S: CopyToHost + ?Sized, D: CopyFromHost + ?Sized>(src: &S, dst: &mut D) {
422 let len: usize = src.len_bytes();
423 assert_eq!(
424 len,
425 dst.len_bytes(),
426 "transfer_buf_into: source is {} bytes, destination is {}",
427 len,
428 dst.len_bytes()
429 );
430 if let Some(bytes) = src.as_host_bytes() {
431 dst.copy_from_host(bytes);
432 return;
433 }
434 if let Some(bytes) = dst.as_host_bytes_mut() {
435 src.copy_to_host(bytes);
436 return;
437 }
438 let mut staging: Vec<u8> = vec![0u8; len];
439 src.copy_to_host(&mut staging);
440 dst.copy_from_host(&staging);
441}
442
443pub trait HostStaged: Backend<ZnxWord = i64, OwnedBuf: CopyToHost + CopyFromHost> {}
449
450impl<BE> HostStaged for BE where BE: Backend<ZnxWord = i64, OwnedBuf: CopyToHost + CopyFromHost> {}
451
452#[macro_export]
459macro_rules! impl_backend_from {
460 (@executor $from:ty, $executor:ty) => { $executor };
461 (@executor $from:ty) => { <$from as poulpy_hal::layouts::Backend>::TaskExecutor };
462 ($be:ty, $from:ty $(, $executor:ty)?) => {
463 impl poulpy_hal::layouts::Backend for $be {
464 const DFT_IS_EXACT: bool = <$from as poulpy_hal::layouts::Backend>::DFT_IS_EXACT;
465
466 type TaskExecutor = poulpy_hal::impl_backend_from!(@executor $from $(, $executor)?);
467 type ZnxWord = <$from as poulpy_hal::layouts::Backend>::ZnxWord;
468 type BigWord = <$from as poulpy_hal::layouts::Backend>::BigWord;
469 type DftWord = <$from as poulpy_hal::layouts::Backend>::DftWord;
470 type OwnedBuf = <$from as poulpy_hal::layouts::Backend>::OwnedBuf;
471 type BufRef<'a> = <$from as poulpy_hal::layouts::Backend>::BufRef<'a>;
472 type BufMut<'a> = <$from as poulpy_hal::layouts::Backend>::BufMut<'a>;
473 type Handle = <$from as poulpy_hal::layouts::Backend>::Handle;
474 type Location = <$from as poulpy_hal::layouts::Backend>::Location;
475
476 fn alloc_bytes(len: usize) -> Self::OwnedBuf {
477 <$from as poulpy_hal::layouts::Backend>::alloc_bytes(len)
478 }
479
480 fn alloc_zeroed_bytes(len: usize) -> Self::OwnedBuf {
481 <$from as poulpy_hal::layouts::Backend>::alloc_zeroed_bytes(len)
482 }
483
484 fn from_host_bytes(bytes: &[u8]) -> Self::OwnedBuf {
485 <$from as poulpy_hal::layouts::Backend>::from_host_bytes(bytes)
486 }
487
488 fn from_bytes(bytes: Vec<u8>) -> Self::OwnedBuf {
489 <$from as poulpy_hal::layouts::Backend>::from_bytes(bytes)
490 }
491
492 fn to_host_bytes(buf: &Self::OwnedBuf) -> Vec<u8> {
493 <$from as poulpy_hal::layouts::Backend>::to_host_bytes(buf)
494 }
495
496 fn copy_to_host(buf: &Self::OwnedBuf, dst: &mut [u8]) {
497 <$from as poulpy_hal::layouts::Backend>::copy_to_host(buf, dst)
498 }
499
500 fn copy_from_host(buf: &mut Self::OwnedBuf, src: &[u8]) {
501 <$from as poulpy_hal::layouts::Backend>::copy_from_host(buf, src)
502 }
503
504 fn copy_view_to_host(buf: &Self::BufRef<'_>, dst: &mut [u8]) {
505 <$from as poulpy_hal::layouts::Backend>::copy_view_to_host(buf, dst)
506 }
507
508 fn copy_host_to_view(buf: &mut Self::BufMut<'_>, src: &[u8]) {
509 <$from as poulpy_hal::layouts::Backend>::copy_host_to_view(buf, src)
510 }
511
512 fn len_bytes(buf: &Self::OwnedBuf) -> usize {
513 <$from as poulpy_hal::layouts::Backend>::len_bytes(buf)
514 }
515
516 fn len_bytes_ref(buf: &Self::BufRef<'_>) -> usize {
517 <$from as poulpy_hal::layouts::Backend>::len_bytes_ref(buf)
518 }
519
520 fn len_bytes_mut(buf: &Self::BufMut<'_>) -> usize {
521 <$from as poulpy_hal::layouts::Backend>::len_bytes_mut(buf)
522 }
523
524 fn view(buf: &Self::OwnedBuf) -> Self::BufRef<'_> {
525 <$from as poulpy_hal::layouts::Backend>::view(buf)
526 }
527
528 fn view_ref<'a, 'b>(buf: &'a Self::BufRef<'b>) -> Self::BufRef<'a>
529 where
530 Self: 'b,
531 {
532 <$from as poulpy_hal::layouts::Backend>::view_ref(buf)
533 }
534
535 fn view_ref_mut<'a, 'b>(buf: &'a Self::BufMut<'b>) -> Self::BufRef<'a>
536 where
537 Self: 'b,
538 {
539 <$from as poulpy_hal::layouts::Backend>::view_ref_mut(buf)
540 }
541
542 fn view_mut_ref<'a, 'b>(buf: &'a mut Self::BufMut<'b>) -> Self::BufMut<'a>
543 where
544 Self: 'b,
545 {
546 <$from as poulpy_hal::layouts::Backend>::view_mut_ref(buf)
547 }
548
549 fn view_mut(buf: &mut Self::OwnedBuf) -> Self::BufMut<'_> {
550 <$from as poulpy_hal::layouts::Backend>::view_mut(buf)
551 }
552
553 fn region(buf: &Self::OwnedBuf, offset: usize, len: usize) -> Self::BufRef<'_> {
554 <$from as poulpy_hal::layouts::Backend>::region(buf, offset, len)
555 }
556
557 fn region_mut(buf: &mut Self::OwnedBuf, offset: usize, len: usize) -> Self::BufMut<'_> {
558 <$from as poulpy_hal::layouts::Backend>::region_mut(buf, offset, len)
559 }
560
561 fn region_ref<'a, 'b>(buf: &'a Self::BufRef<'b>, offset: usize, len: usize) -> Self::BufRef<'a>
562 where
563 Self: 'b,
564 {
565 <$from as poulpy_hal::layouts::Backend>::region_ref(buf, offset, len)
566 }
567
568 fn region_ref_mut<'a, 'b>(buf: &'a Self::BufMut<'b>, offset: usize, len: usize) -> Self::BufRef<'a>
569 where
570 Self: 'b,
571 {
572 <$from as poulpy_hal::layouts::Backend>::region_ref_mut(buf, offset, len)
573 }
574
575 fn region_mut_ref<'a, 'b>(buf: &'a mut Self::BufMut<'b>, offset: usize, len: usize) -> Self::BufMut<'a>
576 where
577 Self: 'b,
578 {
579 <$from as poulpy_hal::layouts::Backend>::region_mut_ref(buf, offset, len)
580 }
581
582 unsafe fn destroy(handle: std::ptr::NonNull<Self::Handle>) {
583 <$from as poulpy_hal::layouts::Backend>::destroy(handle)
584 }
585
586 const SCRATCH_ALIGN: usize = <$from as poulpy_hal::layouts::Backend>::SCRATCH_ALIGN;
592
593 fn bytes_of_vec_znx_dft(n: usize, cols: usize, size: usize) -> usize {
594 <$from as poulpy_hal::layouts::Backend>::bytes_of_vec_znx_dft(n, cols, size)
595 }
596
597 fn bytes_of_vec_znx_big(n: usize, cols: usize, size: usize) -> usize {
598 <$from as poulpy_hal::layouts::Backend>::bytes_of_vec_znx_big(n, cols, size)
599 }
600
601 fn bytes_of_svp_ppol(n: usize, cols: usize) -> usize {
602 <$from as poulpy_hal::layouts::Backend>::bytes_of_svp_ppol(n, cols)
603 }
604
605 fn bytes_of_vmp_pmat(n: usize, rows: usize, cols_in: usize, cols_out: usize, size: usize) -> usize {
606 <$from as poulpy_hal::layouts::Backend>::bytes_of_vmp_pmat(n, rows, cols_in, cols_out, size)
607 }
608
609 fn bytes_of_cnv_pvec_left(n: usize, cols: usize, size: usize) -> usize {
610 <$from as poulpy_hal::layouts::Backend>::bytes_of_cnv_pvec_left(n, cols, size)
611 }
612
613 fn bytes_of_cnv_pvec_right(n: usize, cols: usize, size: usize) -> usize {
614 <$from as poulpy_hal::layouts::Backend>::bytes_of_cnv_pvec_right(n, cols, size)
615 }
616 }
617
618 unsafe impl poulpy_hal::layouts::VecZnxDftLayoutCompatible<$from> for $be {}
621 unsafe impl poulpy_hal::layouts::VecZnxDftLayoutCompatible<$be> for $from {}
622 unsafe impl poulpy_hal::layouts::VecZnxBigLayoutCompatible<$from> for $be {}
623 unsafe impl poulpy_hal::layouts::VecZnxBigLayoutCompatible<$be> for $from {}
624 unsafe impl poulpy_hal::layouts::SvpPPolLayoutCompatible<$from> for $be {}
625 unsafe impl poulpy_hal::layouts::SvpPPolLayoutCompatible<$be> for $from {}
626 unsafe impl poulpy_hal::layouts::VmpPMatLayoutCompatible<$from> for $be {}
627 unsafe impl poulpy_hal::layouts::VmpPMatLayoutCompatible<$be> for $from {}
628 unsafe impl poulpy_hal::layouts::CnvPVecLayoutCompatible<$from> for $be {}
629 unsafe impl poulpy_hal::layouts::CnvPVecLayoutCompatible<$be> for $from {}
630 };
631}
632
633#[derive(Clone, Copy, Debug)]
634pub struct NoiseInfos {
635 pub k: usize,
636 pub sigma: f64,
637 pub bound: f64,
638}
639
640impl NoiseInfos {
641 pub fn new(k: usize, sigma: f64, bound: f64) -> Result<Self> {
642 anyhow::ensure!(sigma.is_sign_positive(), "sigma must be positive");
643 anyhow::ensure!(sigma >= 1.0, "sigma must be greater or equal to 1");
644 anyhow::ensure!(bound >= sigma, "bound: {bound} must be greater or equal to sigma: {sigma}");
645 Ok(Self { k, sigma, bound })
646 }
647
648 pub fn target_limb_and_shift(&self, base2k: usize) -> (usize, u32) {
650 let limb: usize = self.k.div_ceil(base2k) - 1;
651 (limb, ((limb + 1) * base2k - self.k) as u32)
652 }
653}