#![forbid(unsafe_code)]
#![warn(missing_docs)]
mod detect;
mod scan;
mod syntax;
pub use crate::detect::{detect, Detection, THRESHOLD};
pub use crate::scan::{scan, ScanConfig, Span, SpanKind};
pub use crate::syntax::{BlockRule, Dialect, LineRule, StringRule, Syntax};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Options {
pub preserve_lines: bool,
pub keep_blank_lines: bool,
pub trim_trailing_whitespace: bool,
pub keep_shebang: bool,
pub docstrings: bool,
pub strip_markers: bool,
pub dedent: bool,
}
impl Default for Options {
fn default() -> Self {
Options {
preserve_lines: false,
keep_blank_lines: false,
trim_trailing_whitespace: true,
keep_shebang: true,
docstrings: false,
strip_markers: true,
dedent: true,
}
}
}
impl Options {
pub fn preserve_lines(mut self, yes: bool) -> Self {
self.preserve_lines = yes;
self
}
pub fn docstrings(mut self, yes: bool) -> Self {
self.docstrings = yes;
self
}
pub fn keep_markers(mut self, yes: bool) -> Self {
self.strip_markers = !yes;
self
}
fn scan_config(&self) -> ScanConfig {
ScanConfig {
record_strings: false,
docstrings: self.docstrings,
keep_shebang: self.keep_shebang,
}
}
}
pub fn strip_comments(src: &str) -> String {
strip_comments_with(src, &detect(src).syntax(), &Options::default())
}
pub fn strip_comments_with(src: &str, syn: &Syntax, opts: &Options) -> String {
let spans = scan(src, syn, opts.scan_config());
let mut w = LineWriter::new(src.len(), opts);
let mut idx = 0;
for s in spans.iter().filter(|s| s.kind.is_comment()) {
w.push(&src[idx..s.start]);
w.removed(&src[s.start..s.end]);
idx = s.end;
}
w.push(&src[idx..]);
w.finish()
}
pub fn extract_comments(src: &str) -> String {
extract_comments_with(src, &detect(src).syntax(), &Options::default())
}
pub fn extract_comments_with(src: &str, syn: &Syntax, opts: &Options) -> String {
let spans = comments_with(src, syn, opts);
if opts.preserve_lines {
extract_positioned(src, &spans, opts)
} else {
extract_compact(src, &spans, opts)
}
}
pub fn comments(src: &str) -> Vec<Span> {
comments_with(src, &detect(src).syntax(), &Options::default())
}
pub fn comments_with(src: &str, syn: &Syntax, opts: &Options) -> Vec<Span> {
let mut spans = scan(src, syn, opts.scan_config());
spans.retain(|s| s.kind.is_comment());
spans
}
struct LineWriter<'a> {
out: String,
line: String,
dirty: bool,
opts: &'a Options,
}
impl<'a> LineWriter<'a> {
fn new(capacity: usize, opts: &'a Options) -> Self {
LineWriter {
out: String::with_capacity(capacity),
line: String::new(),
dirty: false,
opts,
}
}
fn push(&mut self, text: &str) {
let mut parts = text.split('\n');
if let Some(first) = parts.next() {
self.line.push_str(first);
}
for part in parts {
self.flush(true);
self.line.push_str(part);
}
}
fn removed(&mut self, text: &str) {
self.dirty = true;
if self.opts.preserve_lines {
for _ in text.bytes().filter(|b| *b == b'\n') {
self.flush(true);
}
}
}
fn flush(&mut self, newline: bool) {
let mut content: &str = &self.line;
let cr = newline && content.ends_with('\r');
if cr {
content = &content[..content.len() - 1];
}
if self.dirty && self.opts.trim_trailing_whitespace {
content = content.trim_end_matches([' ', '\t']);
}
let drop_line = self.dirty
&& newline
&& content.is_empty()
&& !self.opts.keep_blank_lines
&& !self.opts.preserve_lines;
if !drop_line {
self.out.push_str(content);
if cr {
self.out.push('\r');
}
if newline {
self.out.push('\n');
}
}
self.line.clear();
self.dirty = false;
}
fn finish(mut self) -> String {
if !self.line.is_empty() || self.dirty {
self.flush(false);
}
self.out
}
}
fn extract_compact(src: &str, spans: &[Span], opts: &Options) -> String {
let mut out = String::new();
let mut prev_end_line: Option<usize> = None;
let mut line = 0usize;
let mut cursor = 0usize;
for s in spans {
let start_line = line + count_newlines(&src[cursor..s.start]);
line = start_line + count_newlines(&src[s.start..s.end]);
cursor = s.end;
let text = render(src, s, opts);
if text.trim().is_empty() {
continue;
}
if prev_end_line.is_some_and(|prev| start_line > prev + 1) {
out.push('\n');
}
out.push_str(&text);
out.push('\n');
prev_end_line = Some(line);
}
out
}
fn extract_positioned(src: &str, spans: &[Span], opts: &Options) -> String {
let total = src.lines().count().max(count_newlines(src));
let mut lines: Vec<String> = vec![String::new(); total];
let mut line = 0usize;
let mut cursor = 0usize;
for s in spans {
let start_line = line + count_newlines(&src[cursor..s.start]);
line = start_line + count_newlines(&src[s.start..s.end]);
cursor = s.end;
for (k, part) in render(src, s, opts).split('\n').enumerate() {
let at = start_line + k;
if at >= lines.len() {
break;
}
if !lines[at].is_empty() {
lines[at].push(' ');
}
lines[at].push_str(part);
}
}
let mut out = lines.join("\n");
if !out.is_empty() {
out.push('\n');
}
out
}
fn render(src: &str, span: &Span, opts: &Options) -> String {
if !opts.strip_markers {
return span.text(src).to_string();
}
let raw = span.inner(src);
let mut lines: Vec<&str> = raw.split('\n').collect();
if let Some(first) = lines.first_mut() {
let repeat = span.text(src).as_bytes()[span.inner_start - span.start - 1];
*first = first.trim_start_matches(repeat as char);
}
let starred = lines.len() > 1
&& lines[1..]
.iter()
.filter(|l| !l.trim().is_empty())
.all(|l| l.trim_start().starts_with('*'));
if starred {
for l in lines[1..].iter_mut() {
let t = l.trim_start();
*l = match t.strip_prefix('*') {
Some(rest) => rest.strip_prefix(' ').unwrap_or(rest),
None => t,
};
}
}
if opts.dedent && lines.len() > 1 && !starred {
let indent = lines[1..]
.iter()
.filter(|l| !l.trim().is_empty())
.map(|l| l.len() - l.trim_start().len())
.min()
.unwrap_or(0);
for l in lines[1..].iter_mut() {
*l = if l.len() >= indent {
&l[indent..]
} else {
l.trim_start()
};
}
}
let mut parts: Vec<&str> = lines.iter().map(|l| l.trim_end()).collect();
if let Some(first) = parts.first_mut() {
*first = first.strip_prefix(' ').unwrap_or(first);
}
while parts.first().is_some_and(|l| l.trim().is_empty()) {
parts.remove(0);
}
while parts.last().is_some_and(|l| l.trim().is_empty()) {
parts.pop();
}
parts.join("\n")
}
fn count_newlines(s: &str) -> usize {
s.bytes().filter(|b| *b == b'\n').count()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strip_removes_comment_only_lines_entirely() {
let src = "a();\n// gone\nb(); // tail\n";
assert_eq!(strip_comments(src), "a();\nb();\n");
}
#[test]
fn preserve_lines_keeps_numbering() {
let src = "a();\n// gone\n/* two\nlines */\nb();\n";
let out = strip_comments_with(
src,
&Dialect::CFamily.syntax(),
&Options::default().preserve_lines(true),
);
assert_eq!(out, "a();\n\n\n\nb();\n");
assert_eq!(out.lines().count(), src.lines().count());
}
#[test]
fn extraction_strips_markers_and_stars() {
let src = "/**\n * One.\n * Two.\n */\nfn x() {} // three\n";
assert_eq!(extract_comments(src), "One.\nTwo.\nthree\n");
let src = "// one\nfn a() {}\nfn b() {}\n// two\n";
assert_eq!(extract_comments(src), "one\n\ntwo\n");
}
#[test]
fn extraction_can_keep_markers() {
let src = "x = 1 # note\n";
let opts = Options::default().keep_markers(true);
assert_eq!(
extract_comments_with(src, &Dialect::Hash.syntax(), &opts),
"# note\n"
);
}
#[test]
fn unknown_syntax_is_a_no_op() {
let src = "just prose, nothing else\n";
assert_eq!(strip_comments(src), src);
assert_eq!(extract_comments(src), "");
}
#[test]
fn input_without_trailing_newline_stays_that_way() {
let syn = Dialect::CFamily.syntax();
let o = Options::default();
assert_eq!(strip_comments_with("x = 1; // c", &syn, &o), "x = 1;");
assert_eq!(strip_comments_with("x = 1;", &syn, &o), "x = 1;");
assert_eq!(strip_comments_with("x = 1;\n// c", &syn, &o), "x = 1;\n");
}
#[test]
fn a_lone_fragment_is_too_little_evidence_to_act_on() {
assert_eq!(strip_comments("x = 1; // c"), "x = 1; // c");
}
#[test]
fn crlf_line_endings_survive() {
let src = "a();\r\n// gone\r\nb(); // tail\r\n";
assert_eq!(strip_comments(src), "a();\r\nb();\r\n");
}
#[test]
fn docstrings_are_opt_in() {
let src = "def f():\n \"\"\"Doc.\"\"\"\n return 1\n# c\n";
let syn = Dialect::Hash.syntax();
assert_eq!(
strip_comments_with(src, &syn, &Options::default()),
"def f():\n \"\"\"Doc.\"\"\"\n return 1\n"
);
let opts = Options::default().docstrings(true);
assert_eq!(
strip_comments_with(src, &syn, &opts),
"def f():\n return 1\n"
);
assert_eq!(extract_comments_with(src, &syn, &opts), "Doc.\n\nc\n");
}
#[test]
fn stripping_never_grows_the_input() {
for src in [
"// a\n",
"#!/bin/sh\n# a\necho \"# not\"\n",
"<!-- x -->\n<p>y</p>\n",
"SELECT 1; -- c\n",
"",
] {
assert!(strip_comments(src).len() <= src.len(), "{src:?}");
}
}
}