use core::fmt;
use std::io::{self, BufRead, Read};
use crate::text::read_full;
pub const MAX_SHEBANG_BYTES: usize = 64 * 1024;
const FIRST_CHUNK: usize = 512;
const CHUNK: usize = 8 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum ShlexError {
NoClosingQuotation,
NoEscapedCharacter,
}
impl fmt::Display for ShlexError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::NoClosingQuotation => "No closing quotation",
Self::NoEscapedCharacter => "No escaped character",
})
}
}
impl std::error::Error for ShlexError {}
pub fn shlex_split(s: &str) -> Result<Vec<String>, ShlexError> {
#[derive(Clone, Copy, PartialEq, Eq)]
enum State {
Whitespace,
Word,
Quote(char),
Escape { in_double: bool },
}
let is_ws = |c: char| matches!(c, ' ' | '\t' | '\r' | '\n');
let mut out = Vec::new();
let mut token = String::new();
let mut quoted = false;
let mut state = State::Whitespace;
for c in s.chars() {
state = match state {
State::Whitespace | State::Word if is_ws(c) => {
if state == State::Word && (!token.is_empty() || quoted) {
out.push(core::mem::take(&mut token));
quoted = false;
}
State::Whitespace
}
State::Whitespace | State::Word => match c {
'\\' => State::Escape { in_double: false },
'\'' | '"' => State::Quote(c),
_ => {
token.push(c);
State::Word
}
},
State::Quote(q) => {
quoted = true;
if c == q {
State::Word
} else if q == '"' && c == '\\' {
State::Escape { in_double: true }
} else {
token.push(c);
state
}
}
State::Escape { in_double } => {
if in_double && c != '"' && c != '\\' {
token.push('\\');
}
token.push(c);
if in_double {
State::Quote('"')
} else {
State::Word
}
}
};
}
match state {
State::Quote(_) => Err(ShlexError::NoClosingQuotation),
State::Escape { .. } => Err(ShlexError::NoEscapedCharacter),
State::Whitespace | State::Word => {
if !token.is_empty() || quoted {
out.push(token);
}
Ok(out)
}
}
}
const fn is_py_ascii_space(b: u8) -> bool {
matches!(b, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c | 0x1c..=0x1f)
}
fn shebang_split(line: &str) -> Vec<String> {
if !line.contains(['\'', '"', '\\']) {
return line
.split([' ', '\t', '\r', '\n'])
.filter(|t| !t.is_empty())
.map(str::to_owned)
.collect();
}
shlex_split(line).unwrap_or_else(|_| {
line.split(|c: char| u8::try_from(c).is_ok_and(is_py_ascii_space))
.filter(|t| !t.is_empty())
.map(str::to_owned)
.collect()
})
}
fn printable(line: &[u8]) -> Option<&str> {
if line
.iter()
.all(|&b| matches!(b, 0x20..=0x7e | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c))
{
core::str::from_utf8(line).ok()
} else {
None
}
}
fn py_strip(s: &str) -> &str {
s.trim_matches(|c: char| u8::try_from(c).is_ok_and(is_py_ascii_space))
}
fn read_line_limited<R: BufRead + ?Sized>(
reader: &mut R,
out: &mut Vec<u8>,
limit: usize,
) -> io::Result<bool> {
loop {
let available = match reader.fill_buf() {
Ok(buf) => buf,
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
};
if available.is_empty() {
return Ok(true);
}
let room = limit.saturating_sub(out.len());
if room == 0 {
return Ok(false);
}
let window = available.get(..room).unwrap_or(available);
if let Some(i) = window.iter().position(|&b| b == b'\n') {
out.extend_from_slice(window.get(..=i).unwrap_or(window));
reader.consume(i + 1);
return Ok(true);
}
let n = window.len();
out.extend_from_slice(window);
reader.consume(n);
}
}
fn with_line<R: BufRead + ?Sized, T>(
reader: &mut R,
limit: usize,
f: impl FnOnce(&[u8]) -> T,
) -> io::Result<Option<(T, usize)>> {
let available = loop {
match reader.fill_buf() {
Ok(buf) => break buf,
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
Err(e) => return Err(e),
}
};
let window = available.get(..limit).unwrap_or(available);
if let Some(i) = window.iter().position(|&b| b == b'\n') {
let out = f(window.get(..=i).unwrap_or(window));
reader.consume(i + 1);
return Ok(Some((out, i + 1)));
}
let mut line = Vec::new();
if !read_line_limited(reader, &mut line, limit)? {
return Ok(None);
}
let used = line.len();
Ok(Some((f(&line), used)))
}
fn tokenize_line(line: &[u8]) -> Option<Vec<String>> {
printable(line).map(|text| shebang_split(py_strip(text)))
}
pub(crate) struct Source<'a, R> {
head: &'a [u8],
rest: R,
buf: Vec<u8>,
pos: usize,
}
impl<'a, R: Read> Source<'a, R> {
pub(crate) fn new(head: &'a [u8], rest: R) -> Self {
Self {
head,
rest,
buf: Vec::new(),
pos: 0,
}
}
}
impl<R: Read> Read for Source<'_, R> {
fn read(&mut self, out: &mut [u8]) -> io::Result<usize> {
let available = self.fill_buf()?;
let n = available.len().min(out.len());
for (o, a) in out.iter_mut().zip(available) {
*o = *a;
}
self.consume(n);
Ok(n)
}
}
impl<R: Read> BufRead for Source<'_, R> {
fn fill_buf(&mut self) -> io::Result<&[u8]> {
if !self.head.is_empty() {
return Ok(self.head);
}
if self.pos >= self.buf.len() {
let size = if self.buf.capacity() == 0 {
FIRST_CHUNK
} else {
CHUNK
};
self.buf.resize(size, 0);
let n = loop {
match self.rest.read(&mut self.buf) {
Ok(n) => break n.min(size),
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
Err(e) => {
self.buf.clear();
self.pos = 0;
return Err(e);
}
}
};
self.buf.truncate(n);
self.pos = 0;
}
Ok(self.buf.get(self.pos..).unwrap_or_default())
}
fn consume(&mut self, amt: usize) {
if self.head.is_empty() {
self.pos = self.pos.saturating_add(amt).min(self.buf.len());
} else {
self.head = self.head.get(amt..).unwrap_or_default();
}
}
}
pub(crate) fn parse_shebang_buf<R: BufRead + ?Sized>(reader: &mut R) -> io::Result<Vec<String>> {
let mut budget = MAX_SHEBANG_BYTES;
let mut magic = [0u8; 2];
if read_full(reader, &mut magic)? != 2 || magic != *b"#!" {
return Ok(Vec::new());
}
budget -= magic.len();
let Some((tokens, used)) = with_line(reader, budget, tokenize_line)? else {
return Ok(Vec::new());
};
budget = budget.saturating_sub(used);
let Some(mut cmd) = tokens else {
return Ok(Vec::new());
};
match cmd.as_slice() {
[env, flag, ..] if env == "/usr/bin/env" && flag == "-S" => {
cmd.drain(..2);
}
[env, ..] if env == "/usr/bin/env" => {
cmd.remove(0);
}
_ => {}
}
if matches!(cmd.as_slice(), [only] if only == "nix-shell") {
return parse_nix_shebang(reader, cmd, budget);
}
Ok(cmd)
}
fn parse_nix_shebang<R: BufRead + ?Sized>(
reader: &mut R,
mut cmd: Vec<String>,
mut budget: usize,
) -> io::Result<Vec<String>> {
loop {
if budget < 2 {
return Ok(cmd);
}
let mut magic = [0u8; 2];
if read_full(reader, &mut magic)? != 2 || magic != *b"#!" {
return Ok(cmd);
}
budget -= 2;
let Some((tokens, used)) = with_line(reader, budget, tokenize_line)? else {
return Ok(cmd);
};
budget = budget.saturating_sub(used);
let Some(tokens) = tokens else {
return Ok(cmd);
};
for pair in tokens.windows(2) {
if let [flag, arg] = pair {
if flag == "-i" {
cmd = vec![arg.clone()];
}
}
}
}
}
pub fn parse_shebang<R: Read>(reader: R) -> io::Result<Vec<String>> {
parse_shebang_buf(&mut Source::new(&[], reader))
}
#[must_use]
pub fn parse_shebang_bytes(content: &[u8]) -> Vec<String> {
let mut reader = content;
parse_shebang_buf(&mut reader).unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
const NONE: Vec<String> = Vec::new();
fn split(s: &str) -> Result<Vec<String>, ShlexError> {
shlex_split(s)
}
#[test]
fn shlex_golden() {
let ok: &[(&str, &[&str])] = &[
("/bin/sh #foo", &["/bin/sh", "#foo"]),
(
"/usr/bin/env python # c",
&["/usr/bin/env", "python", "#", "c"],
),
("\"/usr/bin/my python\" -x", &["/usr/bin/my python", "-x"]),
("a\\ b c", &["a b", "c"]),
("\"a\\\\b\\q\"", &["a\\b\\q"]),
("x\x0by", &["x\x0by"]),
("x\ry", &["x", "y"]),
("a '' b", &["a", "", "b"]),
("''", &[""]),
("a\"b c\"d", &["ab cd"]),
("'a\\b'", &["a\\b"]),
("\"a\\\"b\"", &["a\"b"]),
("", &[]),
(" ", &[]),
("\\'", &["'"]),
];
for (input, expected) in ok {
let expected: Vec<String> = expected.iter().map(|s| (*s).to_owned()).collect();
assert_eq!(split(input), Ok(expected), "{input:?}");
}
assert_eq!(
split("unterminated \"q"),
Err(ShlexError::NoClosingQuotation)
);
assert_eq!(split("'"), Err(ShlexError::NoClosingQuotation));
assert_eq!(split("trail\\"), Err(ShlexError::NoEscapedCharacter));
assert_eq!(split("\"x\\"), Err(ShlexError::NoEscapedCharacter));
assert_eq!(
ShlexError::NoClosingQuotation.to_string(),
"No closing quotation"
);
assert_eq!(shebang_split("unterminated \"q"), ["unterminated", "\"q"]);
assert_eq!(shebang_split("trail\\"), ["trail\\"]);
assert_eq!(shebang_split("a\x0bb 'c"), ["a", "b", "'c"]);
}
#[test]
fn upstream_parse_shebang_cases() {
let cases: &[(&[u8], &[&str])] = &[
(b"", &[]),
(b"#!/usr/bin/python", &["/usr/bin/python"]),
(b"#!/usr/bin/env python", &["python"]),
(b"#! /usr/bin/python", &["/usr/bin/python"]),
(b"#!/usr/bin/foo python", &["/usr/bin/foo", "python"]),
(b"#!\"/path/with spaces/x\" y", &["/path/with spaces/x", "y"]),
(b"#!/path'with/quotes y", &["/path'with/quotes", "y"]),
(b"#! /path'with/quotes y ", &["/path'with/quotes", "y"]),
(b"#! /usr/bin/env nix-shell\n#! nix-shell -i bash -p python", &["bash"]),
(b"#! /usr/bin/env nix-shell\n#! nix-shell -i python -p coreutils", &["python"]),
(b"#! /usr/bin/env nix-shell\n#! nix-shell -p coreutils -i python", &["python"]),
(b"#! /usr/bin/env nix-shell\n#! nix-shell -p coreutils\n#! nix-shell -i python", &["python"]),
(b"#! /usr/bin/env nix-shell\n#!nix-shell -p coreutils\n#!nix-shell -i python", &["python"]),
(b"#! /usr/bin/env nix-shell\n#!\xf9\x93\x01\x42\xcd", &["nix-shell"]),
(b"#! /usr/bin/env nix-shell\n#!\x00\x00\x00\x00", &["nix-shell"]),
(b"#! /usr/bin/nix-shell", &["/usr/bin/nix-shell"]),
(b"#! /usr/bin/env nix-shell", &["nix-shell"]),
(b"#! /usr/bin/env nix-shell non-portable-argument", &["nix-shell", "non-portable-argument"]),
(b"#! /usr/bin/env nix-shell\n#! nix-shell -i", &["nix-shell"]),
(
b"#!/usr/bin/env nix-shell\n#!nix-shell --argstr x \"a -i python3 p\"\n#!nix-shell -p hello\n#!nix-shell -i bash\n#!nix-shell --argstr y \"b -i runhaskell q\"",
&["bash"],
),
(b"\xf9\x93\x01\x42\xcd", &[]),
(b"#!\xf9\x93\x01\x42\xcd", &[]),
(b"#!\x00\x00\x00\x00", &[]),
(b"#!/usr/bin/env -S python -u", &["python", "-u"]),
(b"#!/usr/bin/env", &[]),
(b"#!/usr/bin/env -S", &[]),
];
for (input, expected) in cases {
assert_eq!(
parse_shebang_bytes(input),
*expected,
"{:?}",
String::from_utf8_lossy(input)
);
let expected: Vec<String> = expected.iter().map(|s| (*s).to_owned()).collect();
assert_eq!(parse_shebang(*input).ok(), Some(expected));
}
}
#[test]
fn printable_and_strip_rules() {
assert_eq!(
parse_shebang_bytes(b"#!/bin/sh\x0b-x\x0c\r\n"),
["/bin/sh\x0b-x"]
);
assert_eq!(parse_shebang_bytes(b"#!/bin/sh\r\n"), ["/bin/sh"]);
assert_eq!(parse_shebang_bytes(b"#!/bin/sh\x01\n"), NONE);
assert_eq!(parse_shebang_bytes("#!/bin/sh é\n".as_bytes()), NONE);
assert_eq!(parse_shebang_bytes(b"#!"), NONE);
assert_eq!(parse_shebang_bytes(b"#"), NONE);
assert_eq!(parse_shebang_bytes(b"#! \n"), NONE);
assert_eq!(parse_shebang_bytes(b"#!env python"), ["env", "python"]);
assert_eq!(
parse_shebang_bytes(b"#!/usr/bin/env -S"),
Vec::<String>::new()
);
assert_eq!(
parse_shebang_bytes(b"#!/usr/bin/env -S nix-shell\n#!x -i zsh\n"),
["zsh"]
);
}
#[test]
fn line_limit() {
let mut long = b"#!/bin/sh ".to_vec();
long.resize(MAX_SHEBANG_BYTES + 10, b'a');
assert_eq!(parse_shebang_bytes(&long), NONE);
let mut fits = b"#!/bin/sh ".to_vec();
fits.resize(MAX_SHEBANG_BYTES, b'a');
assert_eq!(parse_shebang_bytes(&fits).len(), 2);
fits.push(b'\n');
assert_eq!(parse_shebang_bytes(&fits), NONE);
}
#[test]
fn bounded_reads_on_endless_input() {
struct Endless {
served: usize,
}
impl Read for Endless {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
for (i, b) in buf.iter_mut().enumerate() {
*b = b"#!".get(self.served + i).copied().unwrap_or(b'a');
}
self.served += buf.len();
Ok(buf.len())
}
}
struct Cycle {
data: Vec<u8>,
pos: usize,
}
impl Read for Cycle {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
for b in buf.iter_mut() {
*b = self.data[self.pos % self.data.len()];
self.pos += 1;
}
Ok(buf.len())
}
}
let mut endless = Endless { served: 0 };
assert!(parse_shebang(&mut endless).is_ok_and(|v| v.is_empty()));
assert!(
endless.served <= MAX_SHEBANG_BYTES + CHUNK,
"served {}",
endless.served
);
let mut nix = Cycle {
data: b"#!/usr/bin/env nix-shell\n".to_vec(),
pos: 0,
};
let cmd = parse_shebang(&mut nix).unwrap_or_default();
assert_eq!(cmd, ["nix-shell"]);
assert!(nix.pos <= MAX_SHEBANG_BYTES + CHUNK, "served {}", nix.pos);
}
#[test]
fn source_reads_head_then_rest() {
let mut src = Source::new(b"#!/usr/bin/env py", &b"thon3 -u\n#! more"[..]);
assert_eq!(
parse_shebang_buf(&mut src).unwrap_or_default(),
["python3", "-u"]
);
let mut rest = String::new();
assert!(src.read_to_string(&mut rest).is_ok());
assert_eq!(rest, "#! more");
}
#[test]
fn read_errors_propagate() {
struct Failing;
impl Read for Failing {
fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {
Err(io::Error::other("boom"))
}
}
assert!(parse_shebang(Failing).is_err());
let mut src = Source::new(b"#!/bin/sh", Failing);
assert!(parse_shebang_buf(&mut src).is_err());
}
}