hyperlight_host/sandbox/
initialized_multi_use.rs

1/*
2Copyright 2025  The Hyperlight Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17use std::sync::{Arc, Mutex};
18
19use hyperlight_common::flatbuffer_wrappers::function_call::{FunctionCall, FunctionCallType};
20use hyperlight_common::flatbuffer_wrappers::function_types::{
21    ParameterValue, ReturnType, ReturnValue,
22};
23use tracing::{Span, instrument};
24
25use super::host_funcs::FunctionRegistry;
26use super::{MemMgrWrapper, WrapperGetter};
27use crate::func::call_ctx::MultiUseGuestCallContext;
28use crate::func::guest_err::check_for_guest_error;
29use crate::func::{ParameterTuple, SupportedReturnType};
30#[cfg(gdb)]
31use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper;
32use crate::hypervisor::handlers::{MemAccessHandlerCaller, OutBHandlerCaller};
33use crate::hypervisor::{Hypervisor, InterruptHandle};
34use crate::mem::ptr::RawPtr;
35use crate::mem::shared_mem::HostSharedMemory;
36use crate::metrics::maybe_time_and_emit_guest_call;
37use crate::sandbox_state::sandbox::{DevolvableSandbox, EvolvableSandbox, Sandbox};
38use crate::sandbox_state::transition::{MultiUseContextCallback, Noop};
39use crate::{HyperlightError, Result};
40
41/// A sandbox that supports being used Multiple times.
42/// The implication of being used multiple times is two-fold:
43///
44/// 1. The sandbox can be used to call guest functions multiple times, each time a
45///    guest function is called the state of the sandbox is reset to the state it was in before the call was made.
46///
47/// 2. A MultiUseGuestCallContext can be created from the sandbox and used to make multiple guest function calls to the Sandbox.
48///    in this case the state of the sandbox is not reset until the context is finished and the `MultiUseSandbox` is returned.
49pub struct MultiUseSandbox {
50    // We need to keep a reference to the host functions, even if the compiler marks it as unused. The compiler cannot detect our dynamic usages of the host function in `HyperlightFunction::call`.
51    pub(super) _host_funcs: Arc<Mutex<FunctionRegistry>>,
52    pub(crate) mem_mgr: MemMgrWrapper<HostSharedMemory>,
53    vm: Box<dyn Hypervisor>,
54    out_hdl: Arc<Mutex<dyn OutBHandlerCaller>>,
55    mem_hdl: Arc<Mutex<dyn MemAccessHandlerCaller>>,
56    dispatch_ptr: RawPtr,
57    #[cfg(gdb)]
58    dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
59}
60
61impl MultiUseSandbox {
62    /// Move an `UninitializedSandbox` into a new `MultiUseSandbox` instance.
63    ///
64    /// This function is not equivalent to doing an `evolve` from uninitialized
65    /// to initialized, and is purposely not exposed publicly outside the crate
66    /// (as a `From` implementation would be)
67    #[instrument(skip_all, parent = Span::current(), level = "Trace")]
68    pub(super) fn from_uninit(
69        host_funcs: Arc<Mutex<FunctionRegistry>>,
70        mgr: MemMgrWrapper<HostSharedMemory>,
71        vm: Box<dyn Hypervisor>,
72        out_hdl: Arc<Mutex<dyn OutBHandlerCaller>>,
73        mem_hdl: Arc<Mutex<dyn MemAccessHandlerCaller>>,
74        dispatch_ptr: RawPtr,
75        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
76    ) -> MultiUseSandbox {
77        Self {
78            _host_funcs: host_funcs,
79            mem_mgr: mgr,
80            vm,
81            out_hdl,
82            mem_hdl,
83            dispatch_ptr,
84            #[cfg(gdb)]
85            dbg_mem_access_fn,
86        }
87    }
88
89    /// Create a new `MultiUseCallContext` suitable for making 0 or more
90    /// calls to guest functions within the same context.
91    ///
92    /// Since this function consumes `self`, the returned
93    /// `MultiUseGuestCallContext` is guaranteed mutual exclusion for calling
94    /// functions within the sandbox. This guarantee is enforced at compile
95    /// time, and no locks, atomics, or any other mutual exclusion mechanisms
96    /// are used at runtime.
97    ///
98    /// If you have called this function, have a `MultiUseGuestCallContext`,
99    /// and wish to "return" it to a `MultiUseSandbox`, call the `finish`
100    /// method on the context.
101    ///
102    /// Example usage (compiled as a "no_run" doctest since the test binary
103    /// will not be found):
104    ///
105    /// ```no_run
106    /// use hyperlight_host::sandbox::{UninitializedSandbox, MultiUseSandbox};
107    /// use hyperlight_common::flatbuffer_wrappers::function_types::{ReturnType, ParameterValue, ReturnValue};
108    /// use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
109    /// use hyperlight_host::sandbox_state::transition::Noop;
110    /// use hyperlight_host::GuestBinary;
111    ///
112    /// // First, create a new uninitialized sandbox, then evolve it to become
113    /// // an initialized, single-use one.
114    /// let u_sbox = UninitializedSandbox::new(
115    ///     GuestBinary::FilePath("some_guest_binary".to_string()),
116    ///     None,
117    /// ).unwrap();
118    /// let sbox: MultiUseSandbox = u_sbox.evolve(Noop::default()).unwrap();
119    /// // Next, create a new call context from the single-use sandbox.
120    /// // After this line, your code will not compile if you try to use the
121    /// // original `sbox` variable.
122    /// let mut ctx = sbox.new_call_context();
123    ///
124    /// // Do a guest call with the context. Assumes that the loaded binary
125    /// // ("some_guest_binary") has a function therein called "SomeGuestFunc"
126    /// // that takes a single integer argument and returns an integer.
127    /// match ctx.call(
128    ///     "SomeGuestFunc",
129    ///     ReturnType::Int,
130    ///     Some(vec![ParameterValue::Int(1)])
131    /// ) {
132    ///     Ok(ReturnValue::Int(i)) => println!(
133    ///         "got successful return value {}",
134    ///         i,
135    ///     ),
136    ///     other => panic!(
137    ///         "failed to get return value as expected ({:?})",
138    ///         other,
139    ///     ),
140    /// };
141    /// // You can make further calls with the same context if you want.
142    /// // Otherwise, `ctx` will be dropped and all resources, including the
143    /// // underlying `MultiUseSandbox`, will be released and no further
144    /// // contexts can be created from that sandbox.
145    /// //
146    /// // If you want to avoid
147    /// // that behavior, call `finish` to convert the context back to
148    /// // the original `MultiUseSandbox`, as follows:
149    /// let _orig_sbox = ctx.finish();
150    /// // Now, you can operate on the original sandbox again (i.e. add more
151    /// // host functions etc...), create new contexts, and so on.
152    /// ```
153    #[instrument(skip_all, parent = Span::current())]
154    pub fn new_call_context(self) -> MultiUseGuestCallContext {
155        MultiUseGuestCallContext::start(self)
156    }
157
158    /// Call a guest function by name, with the given return type and arguments.
159    #[instrument(err(Debug), skip(self, args), parent = Span::current())]
160    pub fn call_guest_function_by_name<Output: SupportedReturnType>(
161        &mut self,
162        func_name: &str,
163        args: impl ParameterTuple,
164    ) -> Result<Output> {
165        maybe_time_and_emit_guest_call(func_name, || {
166            let ret = self.call_guest_function_by_name_no_reset(
167                func_name,
168                Output::TYPE,
169                args.into_value(),
170            );
171            self.restore_state()?;
172            Output::from_value(ret?)
173        })
174    }
175
176    /// This function is kept here for fuzz testing the parameter and return types
177    #[cfg(feature = "fuzzing")]
178    #[instrument(err(Debug), skip(self, args), parent = Span::current())]
179    pub fn call_type_erased_guest_function_by_name(
180        &mut self,
181        func_name: &str,
182        ret_type: ReturnType,
183        args: Vec<ParameterValue>,
184    ) -> Result<ReturnValue> {
185        maybe_time_and_emit_guest_call(func_name, || {
186            let ret = self.call_guest_function_by_name_no_reset(func_name, ret_type, args);
187            self.restore_state()?;
188            ret
189        })
190    }
191
192    /// Restore the Sandbox's state
193    #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
194    pub(crate) fn restore_state(&mut self) -> Result<()> {
195        let mem_mgr = self.mem_mgr.unwrap_mgr_mut();
196        mem_mgr.restore_state_from_last_snapshot()
197    }
198
199    pub(crate) fn call_guest_function_by_name_no_reset(
200        &mut self,
201        function_name: &str,
202        return_type: ReturnType,
203        args: Vec<ParameterValue>,
204    ) -> Result<ReturnValue> {
205        let fc = FunctionCall::new(
206            function_name.to_string(),
207            Some(args),
208            FunctionCallType::Guest,
209            return_type,
210        );
211
212        let buffer: Vec<u8> = fc
213            .try_into()
214            .map_err(|_| HyperlightError::Error("Failed to serialize FunctionCall".to_string()))?;
215
216        self.get_mgr_wrapper_mut()
217            .as_mut()
218            .write_guest_function_call(&buffer)?;
219
220        self.vm.dispatch_call_from_host(
221            self.dispatch_ptr.clone(),
222            self.out_hdl.clone(),
223            self.mem_hdl.clone(),
224            #[cfg(gdb)]
225            self.dbg_mem_access_fn.clone(),
226        )?;
227
228        self.check_stack_guard()?;
229        check_for_guest_error(self.get_mgr_wrapper_mut())?;
230
231        self.get_mgr_wrapper_mut()
232            .as_mut()
233            .get_guest_function_call_result()
234    }
235
236    /// Get a handle to the interrupt handler for this sandbox,
237    /// capable of interrupting guest execution.
238    pub fn interrupt_handle(&self) -> Arc<dyn InterruptHandle> {
239        self.vm.interrupt_handle()
240    }
241}
242
243impl WrapperGetter for MultiUseSandbox {
244    fn get_mgr_wrapper(&self) -> &MemMgrWrapper<HostSharedMemory> {
245        &self.mem_mgr
246    }
247    fn get_mgr_wrapper_mut(&mut self) -> &mut MemMgrWrapper<HostSharedMemory> {
248        &mut self.mem_mgr
249    }
250}
251
252impl Sandbox for MultiUseSandbox {
253    fn check_stack_guard(&self) -> Result<bool> {
254        self.mem_mgr.check_stack_guard()
255    }
256}
257
258impl std::fmt::Debug for MultiUseSandbox {
259    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
260        f.debug_struct("MultiUseSandbox")
261            .field("stack_guard", &self.mem_mgr.get_stack_cookie())
262            .finish()
263    }
264}
265
266impl DevolvableSandbox<MultiUseSandbox, MultiUseSandbox, Noop<MultiUseSandbox, MultiUseSandbox>>
267    for MultiUseSandbox
268{
269    /// Consume `self` and move it back to a `MultiUseSandbox` with previous state.
270    ///
271    /// The purpose of this function is to allow multiple states to be associated with a single MultiUseSandbox.
272    ///
273    /// An implementation such as HyperlightJs or HyperlightWasm can use this to call guest functions to load JS or WASM code and then evolve the sandbox causing state to be captured.
274    /// The new MultiUseSandbox can then be used to call guest functions to execute the loaded code.
275    /// The devolve can be used to return the MultiUseSandbox to the state before the code was loaded. Thus avoiding initialisation overhead
276    #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
277    fn devolve(mut self, _tsn: Noop<MultiUseSandbox, MultiUseSandbox>) -> Result<MultiUseSandbox> {
278        self.mem_mgr
279            .unwrap_mgr_mut()
280            .pop_and_restore_state_from_snapshot()?;
281        Ok(self)
282    }
283}
284
285impl<'a, F>
286    EvolvableSandbox<
287        MultiUseSandbox,
288        MultiUseSandbox,
289        MultiUseContextCallback<'a, MultiUseSandbox, F>,
290    > for MultiUseSandbox
291where
292    F: FnOnce(&mut MultiUseGuestCallContext) -> Result<()> + 'a,
293{
294    /// The purpose of this function is to allow multiple states to be associated with a single MultiUseSandbox.
295    ///
296    /// An implementation such as HyperlightJs or HyperlightWasm can use this to call guest functions to load JS or WASM code and then evolve the sandbox causing state to be captured.
297    /// The new MultiUseSandbox can then be used to call guest functions to execute the loaded code.
298    ///
299    /// The evolve function creates a new MultiUseCallContext which is then passed to a callback function  allowing the
300    /// callback function to call guest functions as part of the evolve process, once the callback function  is complete
301    /// the context is finished using a crate internal method that does not restore the prior state of the Sandbox.
302    /// It then creates a mew  memory snapshot on the snapshot stack and returns the MultiUseSandbox
303    #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
304    fn evolve(
305        self,
306        transition_func: MultiUseContextCallback<'a, MultiUseSandbox, F>,
307    ) -> Result<MultiUseSandbox> {
308        let mut ctx = self.new_call_context();
309        transition_func.call(&mut ctx)?;
310        let mut sbox = ctx.finish_no_reset();
311        sbox.mem_mgr.unwrap_mgr_mut().push_state()?;
312        Ok(sbox)
313    }
314}
315
316#[cfg(test)]
317mod tests {
318    use std::sync::{Arc, Barrier};
319    use std::thread;
320
321    use hyperlight_testing::simple_guest_as_string;
322
323    use crate::func::call_ctx::MultiUseGuestCallContext;
324    use crate::sandbox::{Callable, SandboxConfiguration};
325    use crate::sandbox_state::sandbox::{DevolvableSandbox, EvolvableSandbox};
326    use crate::sandbox_state::transition::{MultiUseContextCallback, Noop};
327    use crate::{GuestBinary, HyperlightError, MultiUseSandbox, Result, UninitializedSandbox};
328
329    // Tests to ensure that many (1000) function calls can be made in a call context with a small stack (1K) and heap(14K).
330    // This test effectively ensures that the stack is being properly reset after each call and we are not leaking memory in the Guest.
331    #[test]
332    fn test_with_small_stack_and_heap() {
333        let mut cfg = SandboxConfiguration::default();
334        cfg.set_heap_size(20 * 1024);
335        cfg.set_stack_size(16 * 1024);
336
337        let sbox1: MultiUseSandbox = {
338            let path = simple_guest_as_string().unwrap();
339            let u_sbox = UninitializedSandbox::new(GuestBinary::FilePath(path), Some(cfg)).unwrap();
340            u_sbox.evolve(Noop::default())
341        }
342        .unwrap();
343
344        let mut ctx = sbox1.new_call_context();
345
346        for _ in 0..1000 {
347            ctx.call::<String>("Echo", "hello".to_string()).unwrap();
348        }
349
350        let sbox2: MultiUseSandbox = {
351            let path = simple_guest_as_string().unwrap();
352            let u_sbox = UninitializedSandbox::new(GuestBinary::FilePath(path), Some(cfg)).unwrap();
353            u_sbox.evolve(Noop::default())
354        }
355        .unwrap();
356
357        let mut ctx = sbox2.new_call_context();
358
359        for i in 0..1000 {
360            ctx.call::<i32>(
361                "PrintUsingPrintf",
362                format!("Hello World {}\n", i).to_string(),
363            )
364            .unwrap();
365        }
366    }
367
368    /// Tests that evolving from MultiUseSandbox to MultiUseSandbox creates a new state
369    /// and devolving from MultiUseSandbox to MultiUseSandbox restores the previous state
370    #[test]
371    fn evolve_devolve_handles_state_correctly() {
372        let sbox1: MultiUseSandbox = {
373            let path = simple_guest_as_string().unwrap();
374            let u_sbox = UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap();
375            u_sbox.evolve(Noop::default())
376        }
377        .unwrap();
378
379        let func = Box::new(|call_ctx: &mut MultiUseGuestCallContext| {
380            call_ctx.call::<i32>("AddToStatic", 5i32)?;
381            Ok(())
382        });
383        let transition_func = MultiUseContextCallback::from(func);
384        let mut sbox2 = sbox1.evolve(transition_func).unwrap();
385        let res: i32 = sbox2.call_guest_function_by_name("GetStatic", ()).unwrap();
386        assert_eq!(res, 5);
387        let mut sbox3: MultiUseSandbox = sbox2.devolve(Noop::default()).unwrap();
388        let res: i32 = sbox3.call_guest_function_by_name("GetStatic", ()).unwrap();
389        assert_eq!(res, 0);
390    }
391
392    #[test]
393    // TODO: Investigate why this test fails with an incorrect error when run alongside other tests
394    #[ignore]
395    #[cfg(target_os = "linux")]
396    fn test_violate_seccomp_filters() -> Result<()> {
397        fn make_get_pid_syscall() -> Result<u64> {
398            let pid = unsafe { libc::syscall(libc::SYS_getpid) };
399            Ok(pid as u64)
400        }
401
402        // First, run  to make sure it fails.
403        {
404            let mut usbox = UninitializedSandbox::new(
405                GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")),
406                None,
407            )
408            .unwrap();
409
410            usbox.register("MakeGetpidSyscall", make_get_pid_syscall)?;
411
412            let mut sbox: MultiUseSandbox = usbox.evolve(Noop::default())?;
413
414            let res: Result<u64> = sbox.call_guest_function_by_name("ViolateSeccompFilters", ());
415
416            #[cfg(feature = "seccomp")]
417            match res {
418                Ok(_) => panic!("Expected to fail due to seccomp violation"),
419                Err(e) => match e {
420                    HyperlightError::DisallowedSyscall => {}
421                    _ => panic!("Expected DisallowedSyscall error: {}", e),
422                },
423            }
424
425            #[cfg(not(feature = "seccomp"))]
426            match res {
427                Ok(_) => (),
428                Err(e) => panic!("Expected to succeed without seccomp: {}", e),
429            }
430        }
431
432        // Second, run with allowing `SYS_getpid`
433        #[cfg(feature = "seccomp")]
434        {
435            let mut usbox = UninitializedSandbox::new(
436                GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")),
437                None,
438            )
439            .unwrap();
440
441            usbox.register_with_extra_allowed_syscalls(
442                "MakeGetpidSyscall",
443                make_get_pid_syscall,
444                vec![libc::SYS_getpid],
445            )?;
446            // ^^^ note, we are allowing SYS_getpid
447
448            let mut sbox: MultiUseSandbox = usbox.evolve(Noop::default())?;
449
450            let res: Result<u64> = sbox.call_guest_function_by_name("ViolateSeccompFilters", ());
451
452            match res {
453                Ok(_) => {}
454                Err(e) => panic!("Expected to succeed due to seccomp violation: {}", e),
455            }
456        }
457
458        Ok(())
459    }
460
461    // We have a secomp specifically for `openat`, but we don't want to crash on `openat`, but rather make sure `openat` returns `EACCES`
462    #[test]
463    #[cfg(target_os = "linux")]
464    fn violate_seccomp_filters_openat() -> Result<()> {
465        // Hostcall to call `openat`.
466        fn make_openat_syscall() -> Result<i64> {
467            use std::ffi::CString;
468
469            let path = CString::new("/proc/sys/vm/overcommit_memory").unwrap();
470
471            let fd_or_err = unsafe {
472                libc::syscall(
473                    libc::SYS_openat,
474                    libc::AT_FDCWD,
475                    path.as_ptr(),
476                    libc::O_RDONLY,
477                )
478            };
479
480            if fd_or_err == -1 {
481                Ok((-std::io::Error::last_os_error().raw_os_error().unwrap()).into())
482            } else {
483                Ok(fd_or_err)
484            }
485        }
486        {
487            // First make sure a regular call to `openat` on /proc/sys/vm/overcommit_memory succeeds
488            let ret = make_openat_syscall()?;
489            assert!(
490                ret >= 0,
491                "Expected openat syscall to succeed, got: {:?}",
492                ret
493            );
494
495            let mut ubox = UninitializedSandbox::new(
496                GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")),
497                None,
498            )
499            .unwrap();
500            ubox.register("Openat_Hostfunc", make_openat_syscall)?;
501
502            let mut sbox = ubox.evolve(Noop::default()).unwrap();
503            let host_func_result = sbox
504                .call_guest_function_by_name::<i64>(
505                    "CallGivenParamlessHostFuncThatReturnsI64",
506                    "Openat_Hostfunc".to_string(),
507                )
508                .expect("Expected to call host function that returns i64");
509
510            if cfg!(feature = "seccomp") {
511                // If seccomp is enabled, we expect the syscall to return EACCES, as setup by our seccomp filter
512                assert_eq!(host_func_result, -libc::EACCES as i64);
513            } else {
514                // If seccomp is not enabled, we expect the syscall to succeed
515                assert!(host_func_result >= 0);
516            }
517        }
518
519        #[cfg(feature = "seccomp")]
520        {
521            // Now let's make sure if we register the `openat` syscall as an extra allowed syscall, it will succeed
522            let mut ubox = UninitializedSandbox::new(
523                GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")),
524                None,
525            )
526            .unwrap();
527            ubox.register_with_extra_allowed_syscalls(
528                "Openat_Hostfunc",
529                make_openat_syscall,
530                [libc::SYS_openat],
531            )?;
532            let mut sbox = ubox.evolve(Noop::default()).unwrap();
533            let host_func_result = sbox
534                .call_guest_function_by_name::<i64>(
535                    "CallGivenParamlessHostFuncThatReturnsI64",
536                    "Openat_Hostfunc".to_string(),
537                )
538                .expect("Expected to call host function that returns i64");
539
540            // should pass regardless of seccomp feature
541            assert!(host_func_result >= 0);
542        }
543
544        Ok(())
545    }
546
547    #[test]
548    fn test_trigger_exception_on_guest() {
549        let usbox = UninitializedSandbox::new(
550            GuestBinary::FilePath(simple_guest_as_string().expect("Guest Binary Missing")),
551            None,
552        )
553        .unwrap();
554
555        let mut multi_use_sandbox: MultiUseSandbox = usbox.evolve(Noop::default()).unwrap();
556
557        let res: Result<()> = multi_use_sandbox.call_guest_function_by_name("TriggerException", ());
558
559        assert!(res.is_err());
560
561        match res.unwrap_err() {
562            HyperlightError::GuestAborted(_, msg) => {
563                // msg should indicate we got an invalid opcode exception
564                assert!(msg.contains("InvalidOpcode"));
565            }
566            e => panic!(
567                "Expected HyperlightError::GuestExecutionError but got {:?}",
568                e
569            ),
570        }
571    }
572
573    #[test]
574    #[ignore] // this test runs by itself because it uses a lot of system resources
575    fn create_1000_sandboxes() {
576        let barrier = Arc::new(Barrier::new(21));
577
578        let mut handles = vec![];
579
580        for _ in 0..20 {
581            let c = barrier.clone();
582
583            let handle = thread::spawn(move || {
584                c.wait();
585
586                for _ in 0..50 {
587                    let usbox = UninitializedSandbox::new(
588                        GuestBinary::FilePath(
589                            simple_guest_as_string().expect("Guest Binary Missing"),
590                        ),
591                        None,
592                    )
593                    .unwrap();
594
595                    let mut multi_use_sandbox: MultiUseSandbox =
596                        usbox.evolve(Noop::default()).unwrap();
597
598                    let res: i32 = multi_use_sandbox
599                        .call_guest_function_by_name("GetStatic", ())
600                        .unwrap();
601
602                    assert_eq!(res, 0);
603                }
604            });
605
606            handles.push(handle);
607        }
608
609        barrier.wait();
610
611        for handle in handles {
612            handle.join().unwrap();
613        }
614    }
615}