use crate::{Edge, EdgeKind, FactSet, Node, NodeKind, Span};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MarkerCategory {
Todo,
Fixme,
Hack,
Stub,
Deferred,
}
impl MarkerCategory {
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::Todo => "todo",
Self::Fixme => "fixme",
Self::Hack => "hack",
Self::Stub => "stub",
Self::Deferred => "deferred",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Marker {
pub category: MarkerCategory,
pub text: String,
pub line: u32,
pub span: Span,
pub anchor: u32,
}
enum Mode {
Substr,
Word,
Phrase,
Annotation,
}
const RULES: &[(&str, MarkerCategory, Mode, bool)] = &[
("todo!(", MarkerCategory::Stub, Mode::Substr, false),
("unimplemented!(", MarkerCategory::Stub, Mode::Substr, false),
("todo", MarkerCategory::Todo, Mode::Phrase, false),
("fixme", MarkerCategory::Fixme, Mode::Phrase, false),
("BUG", MarkerCategory::Fixme, Mode::Word, false),
("HACK", MarkerCategory::Hack, Mode::Word, false),
("XXX", MarkerCategory::Hack, Mode::Word, false),
("bug", MarkerCategory::Fixme, Mode::Annotation, false),
("hack", MarkerCategory::Hack, Mode::Annotation, false),
("xxx", MarkerCategory::Hack, Mode::Annotation, false),
(
"not yet implemented",
MarkerCategory::Stub,
Mode::Phrase,
true,
),
("not implemented", MarkerCategory::Stub, Mode::Phrase, true),
("placeholder", MarkerCategory::Stub, Mode::Phrase, true),
("for now", MarkerCategory::Deferred, Mode::Phrase, true),
("deferred", MarkerCategory::Deferred, Mode::Phrase, true),
("follow-up", MarkerCategory::Deferred, Mode::Phrase, true),
("followup", MarkerCategory::Deferred, Mode::Phrase, true),
("tbd", MarkerCategory::Deferred, Mode::Phrase, true),
];
struct CommentSyntax {
line: &'static [&'static str],
block: Option<(&'static str, &'static str)>,
}
const SLASH_STAR: CommentSyntax = CommentSyntax {
line: &["//"],
block: Some(("/*", "*/")),
};
const HASH: CommentSyntax = CommentSyntax {
line: &["#"],
block: None,
};
const DASH_STAR: CommentSyntax = CommentSyntax {
line: &["--"],
block: Some(("/*", "*/")),
};
const DASH: CommentSyntax = CommentSyntax {
line: &["--"],
block: None,
};
const SEMI: CommentSyntax = CommentSyntax {
line: &[";"],
block: None,
};
fn comment_syntax(path: &str) -> Option<&'static CommentSyntax> {
let ext = path
.rsplit('.')
.next()
.unwrap_or_default()
.to_ascii_lowercase();
let syntax = match ext.as_str() {
"rs" | "c" | "h" | "cc" | "cpp" | "cxx" | "hpp" | "hh" | "js" | "jsx" | "mjs" | "cjs"
| "ts" | "tsx" | "go" | "java" | "kt" | "kts" | "swift" | "scala" | "cs" | "php" | "m"
| "mm" | "rust" | "dart" | "zig" | "v" => &SLASH_STAR,
"py" | "rb" | "sh" | "bash" | "zsh" | "pl" | "pm" | "tcl" | "r" | "nim" => &HASH,
"sql" => &DASH_STAR,
"lua" | "hs" | "elm" | "adb" | "ads" => &DASH,
"el" | "lisp" | "clj" | "cljs" | "cljc" | "scm" | "ss" | "rkt" => &SEMI,
_ => return None,
};
Some(syntax)
}
fn comment_portion(line: &str, syn: &CommentSyntax, in_block: &mut bool) -> String {
let mut out = String::new();
let mut i = 0usize;
while i < line.len() {
if *in_block {
let Some((_, close)) = syn.block else {
*in_block = false;
continue;
};
if let Some(rel) = line[i..].find(close) {
out.push_str(&line[i..i + rel]);
i += rel + close.len();
*in_block = false;
continue;
}
out.push_str(&line[i..]);
break;
}
let mut best: Option<(usize, Option<usize>)> = None;
for lead in syn.line {
if let Some(rel) = line[i..].find(lead) {
let pos = i + rel;
if best.is_none_or(|(bp, _)| pos < bp) {
best = Some((pos, None));
}
}
}
if let Some((open, _)) = syn.block
&& let Some(rel) = line[i..].find(open)
{
let pos = i + rel;
if best.is_none_or(|(bp, _)| pos < bp) {
best = Some((pos, Some(open.len())));
}
}
match best {
None => break,
Some((pos, None)) => {
out.push_str(&line[pos..]);
break;
}
Some((pos, Some(open_len))) => {
*in_block = true;
i = pos + open_len;
}
}
}
out
}
const MAX_TEXT: usize = 200;
const MAX_NAME: usize = 80;
const IGNORE_LINE: &str = "roteiro:ignore";
const IGNORE_FILE: &str = "roteiro:ignore-file";
#[must_use]
pub fn scan_markers(path: &str, bytes: &[u8]) -> Vec<Marker> {
if contains_bytes(bytes, IGNORE_FILE.as_bytes()) {
return Vec::new();
}
let syntax = comment_syntax(path);
let mut in_block = false;
let mut out = Vec::new();
let mut offset: u32 = 0;
for (idx, raw) in bytes.split(|&b| b == b'\n').enumerate() {
let raw_len = u32::try_from(raw.len()).unwrap_or(u32::MAX);
let decoded = String::from_utf8_lossy(raw);
let line = decoded.trim_end_matches('\r');
let comment = match syntax {
Some(syn) => comment_portion(line, syn, &mut in_block),
None => line.to_owned(),
};
if line.contains(IGNORE_LINE) {
offset = offset.saturating_add(raw_len).saturating_add(1);
continue;
}
if let Some(category) = classify(line, &comment) {
let lead = u32::try_from(raw.iter().take_while(|b| b.is_ascii_whitespace()).count())
.unwrap_or(0);
out.push(Marker {
category,
text: cap_chars(line, MAX_TEXT, true),
line: u32::try_from(idx + 1).unwrap_or(u32::MAX),
span: Span::new(offset, offset.saturating_add(raw_len)),
anchor: offset.saturating_add(lead),
});
}
offset = offset.saturating_add(raw_len).saturating_add(1);
}
out
}
fn classify(line: &str, comment: &str) -> Option<MarkerCategory> {
for (needle, category, mode, comment_only) in RULES {
let hay = if *comment_only { comment } else { line };
let hit = match mode {
Mode::Substr => hay.contains(needle),
Mode::Word => find_bounded(hay, needle, false).is_some(),
Mode::Phrase => find_bounded(hay, needle, true).is_some(),
Mode::Annotation => find_annotation(hay, needle).is_some(),
};
if hit {
return Some(*category);
}
}
let t = line.trim_start();
if t.starts_with("- [ ]") || t.starts_with("* [ ]") || t.starts_with("+ [ ]") {
return Some(MarkerCategory::Deferred);
}
None
}
fn find_bounded(hay: &str, needle: &str, ci: bool) -> Option<usize> {
let lowered;
let h: &str = if ci {
lowered = hay.to_ascii_lowercase();
&lowered
} else {
hay
};
let bytes = h.as_bytes();
let mut from = 0;
while let Some(rel) = h[from..].find(needle) {
let start = from + rel;
let end = start + needle.len();
let before_ok = start == 0 || !is_word_byte(bytes[start - 1]);
let after_ok = end >= bytes.len() || !is_word_byte(bytes[end]);
if before_ok && after_ok {
return Some(start);
}
from = start + 1;
}
None
}
fn find_annotation(hay: &str, needle: &str) -> Option<usize> {
let lowered = hay.to_ascii_lowercase();
let bytes = lowered.as_bytes();
let mut from = 0;
while let Some(rel) = lowered[from..].find(needle) {
let start = from + rel;
let end = start + needle.len();
let before_ok = start == 0 || !is_word_byte(bytes[start - 1]);
let after_ok = end < bytes.len() && matches!(bytes[end], b':' | b'(');
if before_ok && after_ok {
return Some(start);
}
from = start + 1;
}
None
}
fn is_word_byte(b: u8) -> bool {
b.is_ascii_alphanumeric() || b == b'_'
}
fn contains_bytes(hay: &[u8], needle: &[u8]) -> bool {
needle.len() <= hay.len() && hay.windows(needle.len()).any(|w| w == needle)
}
fn cap_chars(s: &str, max: usize, trim: bool) -> String {
let s = if trim { s.trim() } else { s };
if s.chars().count() > max {
let mut out: String = s.chars().take(max).collect();
out.push('…');
out
} else {
s.to_owned()
}
}
pub fn augment(facts: &mut FactSet, path: &str, blob_id: &str, bytes: &[u8]) {
for m in scan_markers(path, bytes) {
let key = format!("marker:{path}#{}", m.line);
let container =
innermost_container(&facts.nodes, m.anchor).unwrap_or_else(|| format!("file:{path}"));
facts.nodes.push(Node {
key: key.clone(),
kind: NodeKind::Marker,
name: cap_chars(&m.text, MAX_NAME, false),
path: Some(path.to_owned()),
lang: None,
blob_hash: Some(blob_id.to_owned()),
span: Some(m.span),
meta: serde_json::json!({
"category": m.category.as_str(),
"text": m.text,
"line": m.line,
}),
});
facts
.edges
.push(Edge::derived(container, key, EdgeKind::Contains));
}
}
fn innermost_container(nodes: &[Node], offset: u32) -> Option<String> {
nodes
.iter()
.filter_map(|n| n.span.map(|s| (n, s)))
.filter(|(_, s)| s.start <= offset && offset < s.end)
.min_by(|(a, sa), (b, sb)| (sa.end - sa.start, &a.key).cmp(&(sb.end - sb.start, &b.key)))
.map(|(n, _)| n.key.clone())
}
#[cfg(test)]
mod tests {
use super::{MarkerCategory, augment, scan_markers};
use crate::{EdgeKind, FactSet, Node, NodeKind, Span};
fn categories(src: &str) -> Vec<(u32, MarkerCategory)> {
categories_in("", src)
}
fn categories_in(path: &str, src: &str) -> Vec<(u32, MarkerCategory)> {
scan_markers(path, src.as_bytes())
.into_iter()
.map(|m| (m.line, m.category))
.collect()
}
#[test]
fn detects_each_category() {
let src = "\
// TODO wire this up
let x = todo!();
// FIXME off-by-one
// HACK relies on ordering
// this is a placeholder for now
- [ ] finish the docs
plain line, nothing here
";
let got = categories(src);
assert_eq!(got[0], (1, MarkerCategory::Todo));
assert_eq!(got[1], (2, MarkerCategory::Stub)); assert_eq!(got[2], (3, MarkerCategory::Fixme));
assert_eq!(got[3], (4, MarkerCategory::Hack));
assert_eq!(got[4], (5, MarkerCategory::Stub)); assert_eq!(got[5], (6, MarkerCategory::Deferred)); assert_eq!(got.len(), 6, "the plain line is not a marker");
}
#[test]
fn word_boundaries_avoid_false_positives() {
assert!(categories("mastodon Todos BUGFIX fixmelike").is_empty());
assert_eq!(categories("x // BUG here")[0].1, MarkerCategory::Fixme);
}
#[test]
fn tags_match_mixed_case() {
assert_eq!(categories("// Todo: wire it")[0].1, MarkerCategory::Todo);
assert_eq!(categories("// fixme this path")[0].1, MarkerCategory::Fixme);
assert_eq!(categories("decision TBD")[0].1, MarkerCategory::Deferred);
assert_eq!(categories("// HACK ordering")[0].1, MarkerCategory::Hack);
assert_eq!(categories("note a Bug: crash")[0].1, MarkerCategory::Fixme);
assert_eq!(
categories("// hack(perf): fast path")[0].1,
MarkerCategory::Hack
);
assert!(categories("we fixed a bug in the hack layer").is_empty());
}
#[test]
fn prose_phrases_are_comment_only_in_code_files() {
assert_eq!(
categories_in("a.rs", " // just a placeholder for now\n")[0].1,
MarkerCategory::Stub
);
assert_eq!(
categories_in("a.rs", "let msg = format!(\"loaded {n} for now\");\n")
.first()
.map(|m| m.1),
None,
"a phrase in a string literal is not a marker"
);
assert_eq!(
categories_in("a.rs", "do_thing(); // deferred until v2\n")[0].1,
MarkerCategory::Deferred
);
assert_eq!(
categories_in("a.rs", "let x = 1; /* placeholder */ let y = 2;\n")[0].1,
MarkerCategory::Stub
);
assert!(categories_in("a.rs", "let deferred_tasks = vec![];\n").is_empty());
assert_eq!(
categories_in("a.rs", "let s = \"TODO: not a comment\";\n")[0].1,
MarkerCategory::Todo
);
}
#[test]
fn block_comments_span_lines_in_code_files() {
let got = categories_in(
"a.rs",
"/* a note\n deferred to later\n*/ let placeholder_var = 1;\n",
);
assert_eq!(got.len(), 1);
assert_eq!(got[0], (2, MarkerCategory::Deferred));
}
#[test]
fn prose_files_scan_phrases_everywhere() {
assert_eq!(
categories_in("PLAN.md", "We will defer the audit; deferred to v2.\n")[0].1,
MarkerCategory::Deferred
);
}
#[test]
fn non_slash_comment_syntaxes_gate_phrases() {
assert_eq!(
categories_in("m.py", "x = 1 # placeholder for now\n")[0].1,
MarkerCategory::Stub
);
assert!(categories_in("m.py", "msg = \"deferred until later\"\n").is_empty());
assert_eq!(
categories_in("run.sh", "# deferred to a follow-up\n")[0].1,
MarkerCategory::Deferred
);
assert_eq!(
categories_in("q.sql", "SELECT 1; -- placeholder query\n")[0].1,
MarkerCategory::Stub
);
assert!(categories_in("q.sql", "SELECT 'deferred' AS status;\n").is_empty());
}
#[test]
fn scanning_is_deterministic() {
let src = b"// TODO one\ncode\n// FIXME two\n";
assert_eq!(scan_markers("a.rs", src), scan_markers("a.rs", src));
}
#[test]
fn ignore_directives_suppress_line_and_file() {
let got = categories("// TODO real\n// TODO shush roteiro:ignore\n// FIXME real\n");
assert_eq!(got.len(), 2);
assert_eq!(got[0].0, 1);
assert_eq!(got[1].0, 3);
assert!(
scan_markers(
"a.rs",
b"// TODO x\n// note: roteiro:ignore-file\n// FIXME y\n"
)
.is_empty()
);
}
#[test]
fn augment_attaches_to_innermost_symbol_then_file() {
let mut facts = FactSet::new()
.with_node(Node {
span: Some(Span::new(0, 100)),
..Node::new("file:a.rs", NodeKind::File, "a.rs")
})
.with_node(Node {
span: Some(Span::new(10, 40)),
..Node::new("sym:rust:a.rs#f", NodeKind::Fn, "f")
});
let bytes = b"aaaaaaaaaaaaaaaaaaa\n// FIXME inside fn f\n";
augment(&mut facts, "a.rs", "blob", bytes);
let marker = facts
.nodes
.iter()
.find(|n| n.kind == NodeKind::Marker)
.expect("a marker node");
assert_eq!(marker.meta["category"], "fixme");
let edge = facts
.edges
.iter()
.find(|e| e.kind == EdgeKind::Contains && e.dst == marker.key)
.expect("a contains edge");
assert_eq!(edge.src, "sym:rust:a.rs#f");
}
#[test]
fn augment_attaches_to_symbol_on_its_own_indented_line() {
let mut facts = FactSet::new()
.with_node(Node {
span: Some(Span::new(0, 80)),
..Node::new("file:a.rs", NodeKind::File, "a.rs")
})
.with_node(Node {
span: Some(Span::new(4, 40)),
..Node::new("sym:rust:a.rs#f", NodeKind::Fn, "f")
});
augment(&mut facts, "a.rs", "blob", b" fn f() { // TODO soon }\n");
let marker = facts
.nodes
.iter()
.find(|n| n.kind == NodeKind::Marker)
.unwrap();
let edge = facts
.edges
.iter()
.find(|e| e.kind == EdgeKind::Contains && e.dst == marker.key)
.unwrap();
assert_eq!(edge.src, "sym:rust:a.rs#f");
}
#[test]
fn augment_falls_back_to_file_when_no_symbol_encloses() {
let mut facts = FactSet::new().with_node(Node {
span: Some(Span::new(0, 100)),
..Node::new("file:a.rs", NodeKind::File, "a.rs")
});
augment(&mut facts, "a.rs", "blob", b"// TODO top of file\n");
let marker = facts
.nodes
.iter()
.find(|n| n.kind == NodeKind::Marker)
.unwrap();
let edge = facts
.edges
.iter()
.find(|e| e.kind == EdgeKind::Contains && e.dst == marker.key)
.unwrap();
assert_eq!(edge.src, "file:a.rs");
}
}