argtokens 0.5.0

// Copyright (C) 2021-2022, 2025-2026  Johannes Sasongko <johannes sasongko org>
// SPDX-License-Identifier: MPL-2.0
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

#![doc = include_str!("../README.md")]
// Waiting for <https://rust-lang.github.io/rfcs/3631-rustdoc-cfgs-handling.html>.
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(not(feature = "std"), no_std)]
#![warn(clippy::unused_trait_names)]
#![warn(missing_docs)]
#![warn(unused_results)]

pub mod argument;
#[cfg(test)]
mod tests;
pub mod util;

#[cfg(all(doc, feature = "std"))]
use std::ffi::OsStr;

use crate::argument::{Argument, NotAscii};

/// Argument token, returned by [`ArgTokens::next`] and
/// [`ArgTokens::next_with_check`].
#[derive(Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub enum Token<Arg>
where
	Arg: Argument,
{
	/// Value that occurs as an argument by itself.
	///
	/// If you are following POSIX conventions, you should check whether the
	/// argument is `--`, in which case you should stop retrieving tokens with
	/// [`ArgTokens::next`] or [`ArgTokens::next_with_check`], and switch to
	/// [`ArgTokens::next_value`] for the rest of the arguments.
	Freestanding(Arg),
	/// Short option, for example `o` in `-o`.
	///
	/// This is guaranteed to be an ASCII character.
	Short(u8),
	/// Invalid (non-ASCII) short option.
	///
	/// This returns and consumes all the characters in the current argument,
	/// starting from where this option occurs.
	InvalidShort(Arg),
	/// Long option, for example `opt` in `--opt`.
	///
	/// This is not guaranteed to be ASCII-only and there is no `InvalidLong`
	/// token, but options containing non-ASCII characters are not supported and
	/// should not be individually matched against (they should go through your
	/// normal "invalid option" code path).
	Long(Arg::LongOpt),
	/// Leftover value that is attached to the previous token (a long option).
	///
	/// For example, in `--help=leftover`, if `--help` does not expect a value,
	/// the `leftover` is a hanging value.
	///
	/// Typically you should indicate an error when encountering this.
	/// If your long option expects a value, you should call
	/// [`ArgTokens::next_value`] to obtain and consume the value.
	Hanging(Arg),
}

/// Internal state.
enum State<Arg> {
	/// The next token will be read from the next argument.
	Normal,
	/// The next token may be part of a group of short options.
	InShort {
		/// The remaining part of the argument.
		///
		/// For example, if the current argument is `-foo` and the last-parsed
		/// token is the `f`, `rest` is `oo`.
		///
		/// This can be empty at the end of the argument, which is equivalent to
		/// [`State::Normal`].
		rest: Arg,
	},
	/// The next token is the value of an `=`-attached long option.
	InLong {
		/// The value of the option.
		value: Arg,
	},
}

/// Command-line argument tokenizer.
pub struct ArgTokens<Args>
where
	Args: Iterator<Item: Argument>,
{
	args: core::iter::Peekable<Args>,
	state: State<Args::Item>,
}

impl<Args> ArgTokens<Args>
where
	Args: Iterator<Item: Argument>,
{
	/// Creates a new instance of the tokenizer.
	///
	/// This should not include the program name (usually the first argument).
	/// Note that both the Rust standard library and C standard allow the
	/// argument list to be empty, so do not rely on the first argument to be
	/// present.
	///
	/// # Examples
	///
	/// With `std`:
	///
	/// ```
	/// # use std::ffi::OsString;
	/// # use argtokens::*;
	/// let args = Vec::from_iter(std::env::args_os().skip(1));
	/// let mut tokens = ArgTokens::new(args.iter().map(OsString::as_os_str));
	/// ```
	///
	/// Without `std`, from a C-compatible `main` function:
	///
	/// ```
	/// # use core::ffi::{c_char, c_int, CStr};
	/// # use argtokens::*;
	/// # unsafe {
	/// # let argc: c_int = 0;
	/// # let argv: *const *const c_char = core::ptr::null();
	/// let args = (1..argc).map(|i| CStr::from_ptr(argv.offset(i as isize).read()).to_bytes());
	/// let mut tokens = ArgTokens::new(args);
	/// # }
	/// ```
	#[inline]
	#[must_use]
	pub fn new(args: impl IntoIterator<IntoIter = Args>) -> Self {
		Self {
			args: args.into_iter().peekable(),
			state: State::Normal,
		}
	}

	/// Retrieves a token.
	///
	/// If you want to accept negative numbers as positional arguments, see
	/// [`next_with_check`](ArgTokens::next_with_check).
	///
	/// # Examples
	///
	/// ```
	/// # use std::ffi::{OsStr, OsString};
	/// # use argtokens::*;
	/// #
	/// # let args = ["-ab", "--", "--ab"];
	/// # let args = Vec::from_iter(args.iter().map(OsString::from));
	/// # let mut tokens = ArgTokens::new(args.iter().map(OsString::as_os_str));
	/// #
	/// // Arguments are ["-ab", "--", "--ab"]
	///
	/// assert_eq!(tokens.next(), Some(Token::Short(b'a')));
	/// assert_eq!(tokens.next(), Some(Token::Short(b'b')));
	/// assert_eq!(tokens.next(), Some(Token::Freestanding(OsStr::new("--"))));
	/// assert_eq!(tokens.next(), Some(Token::Long("ab".as_bytes())));
	/// assert_eq!(tokens.next(), None);
	/// ```
	///
	/// # Safety
	///
	/// In addition to Rust's normal safety guarantees, if all the command-line
	/// arguments passed to [`ArgTokens::new`] were encoded by
	/// [`OsStr::as_encoded_bytes`], any value you get from this function is
	/// also safe to be decoded using [`OsStr::from_encoded_bytes_unchecked`].
	#[expect(clippy::should_implement_trait)]
	#[cfg_attr(feature = "inline-next", inline)]
	pub fn next(&mut self) -> Option<Token<Args::Item>> {
		// This loop acts as a goto marker (look out for `continue` statements).
		// Most of the code paths will simply break after the first iteration.
		let token = loop {
			break match self.state {
				State::Normal => {
					let current = self.args.next()?;
					if let Some((opt, value)) = current.parse_long_option() {
						if let Some(value) = value {
							self.state = State::InLong { value };
						}
						Token::Long(opt)
					} else if let Some(rest) = current.parse_short_option_group() {
						self.state = State::InShort { rest };
						continue;
					} else {
						Token::Freestanding(current)
					}
				}
				State::InShort { rest } => match rest.split_short_option() {
					None => {
						self.state = State::Normal;
						continue;
					}
					Some(Err(NotAscii)) => {
						self.state = State::Normal;
						Token::InvalidShort(rest)
					}
					Some(Ok((opt, rest))) => {
						self.state = State::InShort { rest };
						Token::Short(opt)
					}
				},
				State::InLong { value } => {
					self.state = State::Normal;
					Token::Hanging(value)
				}
			};
		};
		Some(token)
	}

	/// Retrieves a token with a preprocessing check.
	///
	/// This method is similar to [`ArgTokens::next`], except this allows you to
	/// perform a check whenever a new argument is pulled from the iterator.
	/// If the check function returns `true`, the argument is forcefully
	/// interpreted as a [`Token::Freestanding`].
	///
	/// You should use this method instead of `next` if you want to interpret
	/// particular option-like arguments as freestanding values instead of as
	/// options.
	/// For example, if your program accepts numbers as positional arguments,
	/// you may wish to parse `-2` as a positional argument instead of as a
	/// short option so that the user doesn't have to type `-- -2`.
	///
	/// # Examples
	///
	/// ```
	/// # use std::ffi::{OsStr, OsString};
	/// # use argtokens::*;
	/// #
	/// # let args = ["-1.2"];
	/// # let args = Vec::from_iter(args.iter().map(OsString::from));
	/// # let mut tokens = ArgTokens::new(args.iter().map(OsString::as_os_str));
	/// #
	/// // Arguments are ["-1.2"]
	///
	/// let numberlike = |s: &OsStr| s.to_str().and_then(|s| s.parse::<f32>().ok()).is_some();
	///
	/// assert_eq!(tokens.next_with_check(numberlike), Some(Token::Freestanding(OsStr::new("-1.2"))));
	/// assert_eq!(tokens.next_with_check(numberlike), None);
	/// ```
	///
	/// # Safety
	///
	/// In addition to Rust's normal safety guarantees, if all the command-line
	/// arguments passed to [`ArgTokens::new`] were encoded by
	/// [`OsStr::as_encoded_bytes`], any value you get from this function is
	/// also safe to be decoded using [`OsStr::from_encoded_bytes_unchecked`].
	#[cfg_attr(feature = "inline-next", inline)]
	pub fn next_with_check(
		&mut self,
		force_freestanding: impl FnOnce(Args::Item) -> bool,
	) -> Option<Token<Args::Item>> {
		let freestanding_forced = !self.peek_value_is_attached()
			&& self.args.peek().copied().is_some_and(force_freestanding);
		if freestanding_forced {
			self.args.next().map(Token::Freestanding)
		} else {
			self.next()
		}
	}

	/// Retrieves a value token.
	///
	/// # Examples
	///
	/// ```
	/// # use std::ffi::{OsStr, OsString};
	/// # use argtokens::*;
	/// #
	/// # let args = ["-ab", "--", "--ab"];
	/// # let args = Vec::from_iter(args.iter().map(OsString::from));
	/// # let mut tokens = ArgTokens::new(args.iter().map(OsString::as_os_str));
	/// #
	/// // Arguments are ["-ab", "--", "--ab"]
	///
	/// tokens.next(); // -a
	/// assert_eq!(tokens.next_value(), Some(OsStr::new("b")));
	/// assert_eq!(tokens.next_value(), Some(OsStr::new("--")));
	/// assert_eq!(tokens.next_value(), Some(OsStr::new("--ab")));
	/// ```
	///
	/// # Safety
	///
	/// In addition to Rust's normal safety guarantees, if all the command-line
	/// arguments passed to [`ArgTokens::new`] were encoded by
	/// [`OsStr::as_encoded_bytes`], any value you get from this function is
	/// also safe to be decoded using [`OsStr::from_encoded_bytes_unchecked`].
	pub fn next_value(&mut self) -> Option<Args::Item> {
		match self.state {
			State::Normal => self.args.next(),
			State::InShort { rest } if rest.is_empty() => self.args.next(),
			State::InShort { rest: value } | State::InLong { value } => {
				self.state = State::Normal;
				Some(value)
			}
		}
	}

	/// Retrieves a value token, but without consuming it.
	///
	/// Similar to [`ArgTokens::next_value`] except the value is not consumed.
	#[must_use]
	pub fn peek_value(&mut self) -> Option<Args::Item> {
		match self.state {
			State::Normal => self.args.peek().copied(),
			State::InShort { rest } if rest.is_empty() => self.args.peek().copied(),
			State::InShort { rest: value } | State::InLong { value } => Some(value),
		}
	}

	/// Returns whether the next value token (as returned by
	/// [`ArgTokens::next_value`] or [`ArgTokens::peek_value`]), if present, is
	/// a value attached to the current option token.
	///
	/// # Examples
	///
	/// ```
	/// # use std::ffi::OsString;
	/// # use argtokens::*;
	/// #
	/// # let args = ["-ab", "-a", "b", "--ab=bb", "--aa", "bb"];
	/// # let args = Vec::from_iter(args.iter().map(OsString::from));
	/// # let mut tokens = ArgTokens::new(args.iter().map(OsString::as_os_str));
	/// #
	/// // Arguments are ["-ab", "-a", "b", "--ab=bb", "--aa", "bb"]
	///
	/// tokens.next(); // -a
	/// assert_eq!(tokens.peek_value_is_attached(), true);
	/// tokens.next_value(); // b
	///
	/// tokens.next(); // -a
	/// assert_eq!(tokens.peek_value_is_attached(), false);
	/// tokens.next_value(); // b
	///
	/// tokens.next(); // --aa
	/// assert_eq!(tokens.peek_value_is_attached(), true);
	/// tokens.next_value(); // bb
	///
	/// tokens.next(); // --aa
	/// assert_eq!(tokens.peek_value_is_attached(), false);
	/// tokens.next_value(); // bb
	/// ```
	#[must_use]
	pub fn peek_value_is_attached(&self) -> bool {
		matches!(self.state, State::InShort { rest } if !rest.is_empty())
			|| matches!(self.state, State::InLong { .. })
	}
}