consolex 0.1.0

//! Probe, create, and release the console of the current process.

use std::io;
use std::str::FromStr;

use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE;
use windows_sys::Win32::System::Console::{
    AllocConsole, AttachConsole, FlushConsoleInputBuffer, FreeConsole, GetConsoleCP,
    GetConsoleMode, GetStdHandle, ReadConsoleW, SetConsoleMode, SetStdHandle,
    ATTACH_PARENT_PROCESS, ENABLE_ECHO_INPUT, ENABLE_LINE_INPUT, STD_ERROR_HANDLE,
    STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
};

use crate::{Error, Result};

/// Desired console state of the current process.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Mode {
    /// Use the existing terminal if available, otherwise create a new
    /// console window.
    Auto,

    /// Always create a new console window.
    Show,

    /// Release the console if one is attached; produce no output.
    Hide,

    /// Leave the console state untouched.
    Keep,
}

/// Returns `true` if the current process is attached to a console.
///
/// Uses `GetConsoleCP` rather than `GetConsoleWindow` so that consoles
/// created with `CREATE_NO_WINDOW` (no visible window) are still detected.
pub fn has_console() -> bool {
    unsafe { GetConsoleCP() != 0 }
}

/// Outcome of attaching a console.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Attached {
    /// Used an existing console (a parent terminal); no new window.
    Existed,
    /// Created a new console window.
    Created,
}

/// Attaches a console to the current process.
///
/// Prefers the parent process's console so that output appears in an
/// existing terminal; falls back to creating a new console window (for
/// example when the binary is double-clicked). Returns [`Attached::Created`]
/// when a new window was made.
pub fn attach() -> Result<Attached> {
    if has_console() {
        return Ok(Attached::Existed);
    }
    // Attach to the parent terminal (cmd, PowerShell, Windows Terminal,
    // ConPTY, ...) so no extra window is shown.
    if unsafe { AttachConsole(ATTACH_PARENT_PROCESS) } != 0 {
        set_std_handles();
        return Ok(Attached::Existed);
    }
    // No parent console: create one of our own (e.g. double-click).
    if unsafe { AllocConsole() } == 0 {
        return Err(Error::Alloc(io::Error::last_os_error()));
    }
    set_std_handles();
    Ok(Attached::Created)
}

/// Creates a new console window for the current process, replacing an
/// existing console if any.
///
/// Unlike [`attach`], which reuses the parent terminal, this always opens a
/// fresh window (for example for `--show`).
pub fn show() -> Result<Attached> {
    // `AllocConsole` fails while a console is already attached.
    if has_console() {
        detach()?;
    }
    if unsafe { AllocConsole() } == 0 {
        return Err(Error::Alloc(io::Error::last_os_error()));
    }
    set_std_handles();
    Ok(Attached::Created)
}

/// Points the standard handles at the process console so that `std::io`
/// (and therefore `println!`/`wait_key`) can use it.
///
/// Opens the console devices directly (`CONIN$`/`CONOUT$`). A naive
/// `SetStdHandle(X, GetStdHandle(X))` is a no-op when the process was
/// launched with redirected standard handles (e.g. via `child_process`
/// pipes), leaving them pointing at the pipe instead of the console.
fn set_std_handles() {
    use windows_sys::Win32::Foundation::GENERIC_READ;
    use windows_sys::Win32::Foundation::GENERIC_WRITE;
    use windows_sys::Win32::Storage::FileSystem::{
        CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING,
    };

    unsafe {
        let mut conin = "CONIN$\0".encode_utf16().collect::<Vec<u16>>();
        let mut conout = "CONOUT$\0".encode_utf16().collect::<Vec<u16>>();

        let input = CreateFileW(
            conin.as_mut_ptr(),
            GENERIC_READ | GENERIC_WRITE,
            FILE_SHARE_READ | FILE_SHARE_WRITE,
            std::ptr::null_mut(),
            OPEN_EXISTING,
            FILE_ATTRIBUTE_NORMAL,
            std::ptr::null_mut(),
        );
        let output = CreateFileW(
            conout.as_mut_ptr(),
            GENERIC_READ | GENERIC_WRITE,
            FILE_SHARE_READ | FILE_SHARE_WRITE,
            std::ptr::null_mut(),
            OPEN_EXISTING,
            FILE_ATTRIBUTE_NORMAL,
            std::ptr::null_mut(),
        );
        if input != INVALID_HANDLE_VALUE {
            SetStdHandle(STD_INPUT_HANDLE, input);
        }
        if output != INVALID_HANDLE_VALUE {
            SetStdHandle(STD_OUTPUT_HANDLE, output);
            SetStdHandle(STD_ERROR_HANDLE, output);
        }
    }
}

/// Releases the console attached to the current process.
///
/// Does nothing when no console is attached.
pub fn detach() -> Result<()> {
    if !has_console() {
        return Ok(());
    }
    if unsafe { FreeConsole() } == 0 {
        return Err(Error::Free(io::Error::last_os_error()));
    }
    Ok(())
}

/// Applies `mode` to the current process.
pub fn set_mode(mode: Mode) -> Result<()> {
    match mode {
        Mode::Auto => attach().map(|_| ()),
        Mode::Show => show().map(|_| ()),
        Mode::Hide => detach(),
        Mode::Keep => Ok(()),
    }
}

/// Initializes the console for the current process according to `mode`.
///
/// Returns `true` when a new console window was created, so the caller can
/// decide whether to pause (e.g. wait for a key) to keep the window open:
///
/// - [`Mode::Auto`] — use the existing terminal if available, otherwise
///   create a new window. Returns `true` only when a window was created.
/// - [`Mode::Show`] — always create a new window; returns `true`.
/// - [`Mode::Hide`] — release the console; returns `false`.
/// - [`Mode::Keep`] — leave the console untouched; returns `false`.
///
/// ```
/// # use consolex::{init, Mode};
/// let created = init(Mode::Auto)?;
/// if created {
///     println!("a new console window was opened");
/// }
/// # Ok::<(), consolex::Error>(())
/// ```
pub fn init(mode: Mode) -> Result<bool> {
    match mode {
        Mode::Auto => Ok(attach()? == Attached::Created),
        Mode::Show => {
            show()?;
            Ok(true)
        }
        Mode::Hide => {
            detach()?;
            Ok(false)
        }
        Mode::Keep => Ok(false),
    }
}

/// Parses a console policy from its string form.
///
/// Accepted values (case-insensitive): `auto`, `show`, `hide`, `keep`.
impl FromStr for Mode {
    type Err = Error;

    fn from_str(s: &str) -> Result<Self> {
        match s.to_ascii_lowercase().as_str() {
            "auto" => Ok(Mode::Auto),
            "show" => Ok(Mode::Show),
            "hide" => Ok(Mode::Hide),
            "keep" => Ok(Mode::Keep),
            _ => Err(Error::Parse(s.to_owned())),
        }
    }
}

/// Builds a [`Mode`] from command-line arguments.
///
/// Defaults to [`Mode::Auto`]. The presence of `--show` or `--hide` selects
/// the corresponding mode; all other arguments are ignored. This lets you
/// collect the mode directly from `std::env::args_os()`:
///
/// ```
/// use consolex::Mode;
///
/// let mode = ["mpv.exe", "--hide"].into_iter().collect::<Mode>();
/// assert_eq!(mode, Mode::Hide);
/// ```
impl<S: AsRef<std::ffi::OsStr>> FromIterator<S> for Mode {
    fn from_iter<T: IntoIterator<Item = S>>(iter: T) -> Self {
        let mut mode = Mode::Auto;
        for arg in iter {
            match arg.as_ref().to_str() {
                Some("--show") => mode = Mode::Show,
                Some("--hide") => mode = Mode::Hide,
                _ => {}
            }
        }
        mode
    }
}

/// Blocks until a key is pressed.
///
/// Returns immediately when there is no console input to read (for example
/// when standard input is redirected to a file or pipe), so it is safe to
/// call in batch contexts.
pub fn wait_key() -> Result<()> {
    let h = unsafe { GetStdHandle(STD_INPUT_HANDLE) };
    if h.is_null() || h == INVALID_HANDLE_VALUE {
        return Ok(());
    }

    // Drop any input queued before the wait so the pause is deliberate.
    unsafe { FlushConsoleInputBuffer(h) };

    let mut mode = 0u32;
    if unsafe { GetConsoleMode(h, &mut mode) } == 0 {
        // Not a console input handle (e.g. redirected stdin).
        return Ok(());
    }

    // Read a single key without line buffering or echo.
    let raw = mode & !(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT);
    let _ = unsafe { SetConsoleMode(h, raw) };

    let mut buf = [0u16; 1];
    let mut read = 0u32;
    let ok =
        unsafe { ReadConsoleW(h, buf.as_mut_ptr() as *mut _, 1, &mut read, std::ptr::null()) };

    // Restore the original console mode, even on failure.
    let _ = unsafe { SetConsoleMode(h, mode) };

    if ok == 0 {
        return Err(Error::Input(io::Error::last_os_error()));
    }
    Ok(())
}