osslsigncode 0.1.1

In-process Rust bindings for the vendored osslsigncode Authenticode implementation
//! Thin RAII around OpenSSL / format pointers used by [`crate::drive`].

use std::ptr::{self, NonNull};

use openssl_sys::{BIO, PKCS7};

use crate::ffi::{BIO_free, PKCS7_free, UI_destroy_method};
use crate::sys::FILE_FORMAT_CTX;

/// An owned, possibly-null pointer freed with `F` on drop or replacement.
pub(crate) struct OwnedPtr<T, F: Fn(*mut T)>(Option<NonNull<T>>, F);

impl<T, F: Fn(*mut T)> OwnedPtr<T, F> {
    pub(crate) fn null(free: F) -> Self {
        Self(None, free)
    }

    pub(crate) fn from_ptr(ptr: *mut T, free: F) -> Self {
        Self(NonNull::new(ptr), free)
    }

    pub(crate) fn as_ptr(&self) -> *mut T {
        self.0.map_or(ptr::null_mut(), NonNull::as_ptr)
    }

    pub(crate) fn is_null(&self) -> bool {
        self.0.is_none()
    }

    pub(crate) fn take(&mut self) -> *mut T {
        self.0.take().map_or(ptr::null_mut(), NonNull::as_ptr)
    }

    pub(crate) fn set(&mut self, ptr: *mut T) {
        if let Some(old) = self.0.take() {
            (self.1)(old.as_ptr());
        }
        self.0 = NonNull::new(ptr);
    }

    pub(crate) fn replace(&mut self, ptr: *mut T) {
        self.0 = NonNull::new(ptr);
    }
}

impl<T, F: Fn(*mut T)> Drop for OwnedPtr<T, F> {
    fn drop(&mut self) {
        if let Some(ptr) = self.0.take() {
            (self.1)(ptr.as_ptr());
        }
    }
}

/// Owned `PKCS7 *` freed with `PKCS7_free`.
pub(crate) type OwnedPkcs7 = OwnedPtr<PKCS7, fn(*mut PKCS7)>;

pub(crate) fn owned_pkcs7_null() -> OwnedPkcs7 {
    OwnedPtr::null(|ptr| unsafe { PKCS7_free(ptr) })
}

/// Format context cleaned with `format->ctx_cleanup`.
pub(crate) type FormatCtx = OwnedPtr<FILE_FORMAT_CTX, fn(*mut FILE_FORMAT_CTX)>;

pub(crate) fn format_ctx_null() -> FormatCtx {
    OwnedPtr::null(cleanup_format_ctx)
}

pub(crate) fn format_ctx_from_ptr(ptr: *mut FILE_FORMAT_CTX) -> FormatCtx {
    OwnedPtr::from_ptr(ptr, cleanup_format_ctx)
}

fn cleanup_format_ctx(ctx: *mut FILE_FORMAT_CTX) {
    unsafe {
        let format = (*ctx).format;
        if !format.is_null()
            && let Some(cleanup) = (*format).ctx_cleanup
        {
            cleanup(ctx);
        }
    }
}

/// UI method installed into the promoted `ui_method` global for one run.
pub(crate) struct UiGuard;

impl UiGuard {
    pub(crate) unsafe fn install() -> Option<Self> {
        unsafe {
            let method = crate::ffi::ui_osslsigncode();
            if method.is_null() {
                return None;
            }
            crate::ffi::ui_method = method;
            Some(Self)
        }
    }
}

impl Drop for UiGuard {
    fn drop(&mut self) {
        unsafe {
            let method = crate::ffi::ui_method;
            crate::ffi::ui_method = ptr::null_mut();
            if !method.is_null() {
                UI_destroy_method(method);
            }
        }
    }
}

/// Free a BIO chain popped one node at a time (hash filter + outdata).
pub(crate) unsafe fn free_bio_chain(mut head: *mut BIO, outdata: *mut BIO) {
    let mut outdata_in_hash = false;
    while !head.is_null() {
        let tail = unsafe { crate::ffi::BIO_pop(head) };
        if head == outdata {
            outdata_in_hash = true;
        }
        unsafe { BIO_free(head) };
        head = tail;
    }
    if !outdata_in_hash && !outdata.is_null() {
        unsafe { openssl_sys::BIO_free_all(outdata) };
    }
}