use crate::Position;
use crate::diff_parser::LineChange;
use crate::fs::{FileSystem, PathChecker};
use crate::language_parsers::{LanguageParser, LanguageParsers};
use crate::repo_path::RepoPath;
use anyhow::{Context, anyhow, bail};
use serde_repr::Serialize_repr;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::ffi::OsString;
use std::ops::{Range, RangeInclusive};
use std::path::Path;
use std::str::FromStr;
use strum_macros::EnumString;
const UNNAMED_BLOCK_LABEL: &str = "(unnamed)";
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Block {
pub(crate) attributes: HashMap<String, String>,
pub(crate) start_tag_position_range: RangeInclusive<Position>,
pub(crate) content_bytes_range: Range<usize>,
pub(crate) content_position_range: Range<Position>,
}
impl PartialOrd for Block {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Block {
fn cmp(&self, other: &Self) -> Ordering {
self.start_tag_position_range
.start()
.cmp(other.start_tag_position_range.start())
}
}
impl Block {
pub(crate) fn new(
attributes: HashMap<String, String>,
start_tag_position_range: RangeInclusive<Position>,
content_range: Range<usize>,
content_position_range: Range<Position>,
) -> Self {
Self {
attributes,
start_tag_position_range,
content_bytes_range: content_range,
content_position_range,
}
}
fn content_intersects_with_any(&self, line_changes: &[LineChange]) -> bool {
Self::changes_on_lines(
line_changes,
self.content_position_range.start.line,
self.content_position_range.end.line,
)
.any(|line_change| {
Self::intersects_with_line_change(&self.content_position_range, line_change)
})
}
fn start_tag_intersects_with_any(&self, line_changes: &[LineChange]) -> bool {
Self::changes_on_lines(
line_changes,
self.start_tag_position_range.start().line,
self.start_tag_position_range.end().line,
)
.any(|line_change| {
Self::intersects_with_line_change_inclusive(&self.start_tag_position_range, line_change)
})
}
fn changes_on_lines(
line_changes: &[LineChange],
first_line: usize,
last_line: usize,
) -> impl Iterator<Item = &LineChange> {
let first = line_changes.partition_point(|line_change| line_change.line < first_line);
line_changes[first..]
.iter()
.take_while(move |line_change| line_change.line <= last_line)
}
fn intersects_with_line_change_inclusive(
position_range: &RangeInclusive<Position>,
line_change: &LineChange,
) -> bool {
if line_change.line < position_range.start().line {
return false;
}
if line_change.line > position_range.end().line {
return false;
}
if let Some(ranges) = &line_change.ranges {
let start_character = if line_change.line == position_range.start().line {
position_range.start().character - 1 } else {
0
};
let end_character = if line_change.line < position_range.end().line {
usize::MAX
} else {
position_range.end().character - 1 };
ranges
.binary_search_by(|range| {
if range.end > start_character && range.start <= end_character {
Ordering::Equal
} else if range.end <= start_character {
Ordering::Less
} else {
Ordering::Greater
}
})
.is_ok()
} else {
true
}
}
fn intersects_with_line_change(
position_range: &Range<Position>,
line_change: &LineChange,
) -> bool {
if line_change.line < position_range.start.line {
return false;
}
if line_change.line > position_range.end.line {
return false;
}
if let Some(ranges) = &line_change.ranges {
let start_character = if line_change.line == position_range.start.line {
position_range.start.character - 1 } else {
0
};
let end_character = if line_change.line < position_range.end.line {
usize::MAX
} else {
position_range.end.character - 1 };
ranges
.binary_search_by(|range| {
if range.end > start_character && range.start < end_character {
Ordering::Equal
} else if range.end <= start_character {
Ordering::Less
} else {
Ordering::Greater
}
})
.is_ok()
} else {
true
}
}
pub(crate) fn name(&self) -> Option<&str> {
self.attributes.get("name").map(String::as_str)
}
pub(crate) fn name_display(&self) -> &str {
self.name().unwrap_or(UNNAMED_BLOCK_LABEL)
}
pub(crate) fn content<'source>(&self, source: &'source str) -> &'source str {
&source[self.content_bytes_range.clone()]
}
pub(crate) fn content_position(
&self,
content_line_idx: usize,
column_offset: usize,
) -> Position {
let line_start_column = if content_line_idx == 0 {
self.content_position_range.start.character
} else {
1
};
Position::new(
self.content_position_range.start.line + content_line_idx,
line_start_column + column_offset,
)
}
pub(crate) fn severity(&self) -> anyhow::Result<BlockSeverity> {
self.attributes
.get("severity")
.map_or(Ok(BlockSeverity::Error), |s| {
BlockSeverity::from_str(s.as_str())
.context(format!("Invalid \"severity\" attribute value \"{}\"", s))
})
}
}
#[derive(Clone, Copy, Serialize_repr, EnumString, Debug, PartialEq)]
#[strum(ascii_case_insensitive)]
#[repr(u8)]
pub enum BlockSeverity {
Error = 1,
Warning = 2,
Info = 3,
Hint = 4,
}
#[derive(Debug)]
pub struct FileBlocks {
pub(crate) file_content: String,
pub(crate) blocks_with_context: Vec<BlockWithContext>,
}
impl FileBlocks {
fn is_empty(&self) -> bool {
self.blocks_with_context.is_empty()
}
pub(crate) fn to_serializable_report(&self) -> Vec<serde_json::Value> {
self.blocks_with_context
.iter()
.map(|block| {
serde_json::json!({
"name": block.block.name_display(),
"line": block.block.start_tag_position_range.start().line,
"column": block.block.start_tag_position_range.start().character,
"is_content_modified": block.is_content_modified,
"attributes": block.block.attributes,
})
})
.collect()
}
}
#[derive(Debug, Clone)]
pub struct BlockWithContext {
pub(crate) block: Block,
pub(crate) is_start_tag_modified: bool,
pub(crate) is_content_modified: bool,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct ScanStats {
pub files_scanned: usize,
pub files_skipped: usize,
}
#[derive(Debug, Default)]
pub struct ParsedBlocks {
pub blocks: HashMap<RepoPath, FileBlocks>,
pub stats: ScanStats,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScanMode {
All,
OnlyChanged,
}
pub fn parse_blocks(
line_changes_by_file: &HashMap<RepoPath, Vec<LineChange>>,
scan_mode: ScanMode,
file_system: &impl FileSystem,
path_checker: &impl PathChecker,
parsers: &LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> anyhow::Result<ParsedBlocks> {
ensure_diff_has_valid_paths(
line_changes_by_file,
file_system,
path_checker,
parsers,
extra_file_extensions,
)?;
match scan_mode {
ScanMode::All => parse_all_files(
line_changes_by_file,
file_system,
path_checker,
parsers,
extra_file_extensions,
),
ScanMode::OnlyChanged => parse_changed_files(
line_changes_by_file,
file_system,
path_checker,
parsers,
extra_file_extensions,
),
}
}
fn ensure_diff_has_valid_paths(
line_changes_by_file: &HashMap<RepoPath, Vec<LineChange>>,
file_system: &impl FileSystem,
path_checker: &impl PathChecker,
parsers: &LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> anyhow::Result<()> {
let mut invalid_path: Option<&RepoPath> = None;
for file_path in line_changes_by_file.keys() {
if !path_checker.should_allow(file_path) || path_checker.should_ignore(file_path) {
continue;
}
if parser_for_file_path(file_path.as_path(), parsers, extra_file_extensions).is_none() {
continue;
}
if file_system.exists(file_path.as_path()) {
return Ok(());
}
invalid_path = Some(invalid_path.map_or(file_path, |lowest| lowest.min(file_path)));
}
match invalid_path {
None => Ok(()),
Some(file_path) => Err(anyhow!("{}", invalid_diff_target_message(file_path))),
}
}
fn invalid_diff_target_message(file_path: &RepoPath) -> String {
format!("diff target \"{file_path}\" does not exist in the repository root.")
}
fn parse_all_files(
line_changes_by_file: &HashMap<RepoPath, Vec<LineChange>>,
file_system: &impl FileSystem,
path_checker: &impl PathChecker,
parsers: &LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> anyhow::Result<ParsedBlocks> {
let mut parsed = ParsedBlocks::default();
for repo_path_result in file_system.walk() {
let file_path =
repo_path_result.map_err(|err| anyhow!("Failed to walk directory: {err}"))?;
if !path_checker.should_allow(&file_path) || path_checker.should_ignore(&file_path) {
continue;
}
let line_changes = line_changes_by_file
.get(&file_path)
.map_or(&[][..], Vec::as_slice);
let file_blocks = parse_file(
file_system,
file_path.as_path(),
line_changes,
every_block,
parsers,
extra_file_extensions,
)?;
record_parsed_file(&mut parsed, file_path, file_blocks);
}
Ok(parsed)
}
fn parse_changed_files(
line_changes_by_file: &HashMap<RepoPath, Vec<LineChange>>,
file_system: &impl FileSystem,
path_checker: &impl PathChecker,
parsers: &LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> anyhow::Result<ParsedBlocks> {
let mut parsed = ParsedBlocks::default();
for (file_path, line_changes) in line_changes_by_file {
if !path_checker.should_allow(file_path) || path_checker.should_ignore(file_path) {
continue;
}
let file_blocks = parse_file(
file_system,
file_path.as_path(),
line_changes,
modified_blocks,
parsers,
extra_file_extensions,
)
.map_err(|error| {
if file_system.exists(file_path.as_path()) {
error
} else {
error.context(invalid_diff_target_message(file_path))
}
})?;
record_parsed_file(&mut parsed, file_path.clone(), file_blocks);
}
Ok(parsed)
}
fn record_parsed_file(
parsed: &mut ParsedBlocks,
file_path: RepoPath,
file_blocks: Option<FileBlocks>,
) {
match file_blocks {
Some(file_blocks) => {
parsed.stats.files_scanned += 1;
if !file_blocks.is_empty() {
parsed.blocks.insert(file_path, file_blocks);
}
}
None => parsed.stats.files_skipped += 1,
}
}
pub fn every_block(_block: &BlockWithContext) -> bool {
true
}
pub fn modified_blocks(block: &BlockWithContext) -> bool {
block.is_content_modified || block.is_start_tag_modified
}
pub fn parse_file(
file_system: &impl FileSystem,
file_path: &Path,
line_changes: &[LineChange],
block_predicate: impl Fn(&BlockWithContext) -> bool,
parsers: &LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> anyhow::Result<Option<FileBlocks>> {
let parser = match parser_for_file_path(file_path, parsers, extra_file_extensions) {
None => return Ok(None),
Some(p) => p,
};
let source_code = file_system.read_to_string(file_path)?;
let mut names_seen: HashMap<String, Position> = HashMap::new();
let blocks_with_context = parser
.lock()
.expect("no active locks")
.parse(&source_code)
.filter_map(|block| {
let block = match block {
Ok(block) => block,
Err(error) => return Some(Err(error)),
};
if let Err(err) = validate_block_syntax(&block, file_path) {
return Some(Err(err));
}
if let Err(err) = reject_duplicate_name(&block, file_path, &mut names_seen) {
return Some(Err(err));
}
let block_with_context = BlockWithContext {
is_content_modified: block.content_intersects_with_any(line_changes),
is_start_tag_modified: block.start_tag_intersects_with_any(line_changes),
block,
};
block_predicate(&block_with_context).then_some(Ok(block_with_context))
})
.collect::<anyhow::Result<Vec<_>>>()
.context(format!("Failed to parse file {file_path:?}"))?;
Ok(Some(FileBlocks {
file_content: source_code,
blocks_with_context,
}))
}
const RECOGNIZED_ATTRIBUTES: &[&str] = &[
"affects",
"check-ai",
"check-ai-pattern",
"check-lua",
"check-lua-pattern",
"check-lua-timeout",
"keep-sorted",
"keep-sorted-format",
"keep-sorted-pattern",
"keep-unique",
"keep-unique-pattern",
"line-count",
"line-pattern",
"name",
"same-as",
"same-as-format",
"same-as-mode",
"same-as-pattern",
"severity",
];
fn validate_block_syntax(block: &Block, file_path: &Path) -> anyhow::Result<()> {
for attr in block.attributes.keys() {
if !RECOGNIZED_ATTRIBUTES.contains(&attr.as_str()) {
bail!(
"Block {}:{} at line {}, column {} contains unrecognized attribute `{}`",
file_path.display(),
block.name_display(),
block.start_tag_position_range.start().line,
block.start_tag_position_range.start().character,
attr,
);
}
}
block.severity().map(|_| ()).context(format!(
"Block {}:{} at line {}, column {} contains unrecognized severity value",
file_path.display(),
block.name_display(),
block.start_tag_position_range.start().line,
block.start_tag_position_range.start().character,
))
}
fn reject_duplicate_name(
block: &Block,
file_path: &Path,
names_seen: &mut HashMap<String, Position>,
) -> anyhow::Result<()> {
let Some(name) = block.name() else {
return Ok(());
};
let position = block.start_tag_position_range.start();
match names_seen.entry(name.to_string()) {
Entry::Occupied(entry) => {
bail!(
"Block {}:{} at line {}, column {} duplicates the name of the block at line {}, column {}",
file_path.display(),
name,
position.line,
position.character,
entry.get().line,
entry.get().character,
)
}
Entry::Vacant(entry) => {
entry.insert(position.clone());
Ok(())
}
}
}
fn parser_for_file_path<'p>(
file_path: &Path,
parsers: &'p LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> Option<&'p LanguageParser> {
let file_name = file_path.file_name()?.to_str()?;
for (i, _) in file_name.match_indices('.').rev() {
let extension = &file_name[i + 1..];
let ext_os = OsString::from(extension);
if let Some(parser) = try_parser_for_extension(&ext_os, parsers, extra_file_extensions) {
return Some(parser);
}
}
try_parser_for_extension(&OsString::from(file_name), parsers, extra_file_extensions)
}
fn try_parser_for_extension<'p>(
extension: &OsString,
parsers: &'p LanguageParsers,
extra_file_extensions: &HashMap<OsString, OsString>,
) -> Option<&'p LanguageParser> {
let ext = if let Some(ext) = extra_file_extensions.get(extension) {
ext
} else {
extension
};
parsers.get(ext)
}
#[cfg(test)]
mod block_severity_from_str_tests {
use crate::Position;
use crate::blocks::{Block, BlockSeverity};
use std::collections::HashMap;
pub(crate) fn new_empty_block_with_severity(severity: &str) -> Block {
Block::new(
HashMap::from([("severity".into(), severity.into())]),
Position::new(0, 0)..=Position::new(0, 0),
0..0,
Position::new(0, 0)..Position::new(0, 0),
)
}
#[test]
fn block_with_valid_severity_attribute_returns_correct_severity() {
let block = new_empty_block_with_severity("warning");
assert_eq!(block.severity().unwrap(), BlockSeverity::Warning);
}
#[test]
fn block_with_mixed_case_severity_attribute_returns_correct_severity() {
let block = new_empty_block_with_severity("InFo");
assert_eq!(block.severity().unwrap(), BlockSeverity::Info);
}
#[test]
fn block_without_severity_attribute_returns_error_severity() {
let block = Block::new(
HashMap::new(),
Position::new(0, 0)..=Position::new(0, 0),
0..0,
Position::new(0, 0)..Position::new(0, 0),
);
assert_eq!(block.severity().unwrap(), BlockSeverity::Error);
}
#[test]
fn block_with_invalid_severity_attribute_returns_error() {
let block = new_empty_block_with_severity("warn");
assert!(block.severity().is_err());
}
}
#[cfg(test)]
mod parse_blocks_tests {
use crate::blocks::*;
use crate::fs::test_utils::{FakeFileSystem, FakePathChecker};
use crate::language_parsers::language_parsers;
use crate::test_utils::{self};
use std::collections::HashSet;
fn line_change(line: usize) -> LineChange {
LineChange { line, ranges: None }
}
#[test]
fn parse_blocks_counts_scanned_and_skipped_files() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([
(
"with_blocks.py".to_string(),
"# <block keep-sorted=\"asc\">\n'a'\n# </block>\n".to_string(),
),
("without_blocks.py".to_string(), "x = 1\n".to_string()),
("notes.unknown".to_string(), "not a language\n".to_string()),
]));
let parsers = language_parsers()?;
let parsed = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?;
assert_eq!(parsed.blocks.len(), 1);
assert_eq!(parsed.stats.files_scanned, 2);
assert_eq!(parsed.stats.files_skipped, 1);
Ok(())
}
#[test]
fn diff_targets_mode_returns_only_blocks_with_modified_start_tag_or_content()
-> anyhow::Result<()> {
let content_a = r#"
/* <block name="first"> */ let foo = "bar"; // </block>
/* <block name="second"> */ let foo = "baz"; // </block>
/* <block name="third"> */ third block /* </block> */
/* <block name="fourth"> */ fourth block // </block>
/* <block name="fifth"> */ let foo="boo"; // </block>
/* <block
name="sixth"
keep-sorted="asc"> */ block six // </block>
/* <block name="seventh"
keep-sorted="asc"> */ block seven
// </block>
/* <block name="eighth"
keep-sorted="asc"> */ block eight
// </block>
// <block name="ninth">
block nine
// </block>
// <block name="tenth">
block ten // </block>
// <block name="eleventh">
block eleven // </block>
// <block name="twelfth">
twelve /*
Some comment.
</block> */
"#;
let content_b = "/* <block name=\"first\"> */let foo = \"bar\"; // </block>";
let file_system = FakeFileSystem::new(HashMap::from([
("a.rs".to_string(), content_a.to_string()),
("b.rs".to_string(), content_b.to_string()),
]));
let line_changes = HashMap::from([
(
RepoPath::from_reference("a.rs")?,
vec![
line_change(1), LineChange {
line: 2,
ranges: Some(vec![
test_utils::substr_range(
content_a.lines().nth(1).unwrap(),
"/* <block ",
),
test_utils::substr_range(
content_a.lines().nth(1).unwrap(),
"name=\"first\"> */",
),
]), },
LineChange {
line: 3,
ranges: Some(vec![
test_utils::substr_range(
content_a.lines().nth(2).unwrap(),
"/* <block name=\"second\"> */ let foo ",
),
test_utils::substr_range(
content_a.lines().nth(2).unwrap(),
" = \"baz\"; ",
),
]),
},
LineChange {
line: 4,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(3).unwrap(),
" third block ",
)]), },
LineChange {
line: 5,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(4).unwrap(),
" fourth block // </block>",
)]), },
LineChange {
line: 6,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(5).unwrap(),
" </block>",
)]), },
LineChange {
line: 8,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(7).unwrap(),
"name=\"sixth\"",
)]), },
LineChange {
line: 11,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(10).unwrap(),
"keep-sorted=\"asc\"> */",
)]), },
LineChange {
line: 14,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(13).unwrap(),
" block eight",
)]), },
LineChange {
line: 17,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(16).unwrap(),
"block nine",
)]), },
LineChange {
line: 20,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(19).unwrap(),
"block ten ",
)]), },
LineChange {
line: 22,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(21).unwrap(),
" </block>",
)]), },
LineChange {
line: 25,
ranges: Some(vec![test_utils::substr_range(
content_a.lines().nth(24).unwrap(),
"Some comment.",
)]), },
],
),
(
RepoPath::from_reference("b.rs")?,
vec![LineChange {
line: 1,
ranges: Some(vec![test_utils::substr_range(
content_b.lines().next().unwrap(),
"let foo = \"bar\"; ",
)]), }],
),
]);
let parsers = language_parsers()?;
let blocks_by_file = parse_blocks(
&line_changes,
ScanMode::OnlyChanged,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks_by_file.len(), 2);
let blocks_a = &blocks_by_file[&RepoPath::from_reference("a.rs")?].blocks_with_context;
assert_eq!(blocks_a.len(), 9);
let first = &blocks_a[0];
assert_eq!(first.block.name(), Some("first"));
assert!(first.is_start_tag_modified);
assert!(!first.is_content_modified);
let second = &blocks_a[1];
assert_eq!(second.block.name(), Some("second"));
assert!(second.is_start_tag_modified);
assert!(second.is_content_modified);
let third = &blocks_a[2];
assert_eq!(third.block.name(), Some("third"));
assert!(!third.is_start_tag_modified);
assert!(third.is_content_modified);
let fourth = &blocks_a[3];
assert_eq!(fourth.block.name(), Some("fourth"));
assert!(!fourth.is_start_tag_modified);
assert!(fourth.is_content_modified);
let sixth = &blocks_a[4];
assert_eq!(sixth.block.name(), Some("sixth"));
assert!(sixth.is_start_tag_modified);
assert!(!sixth.is_content_modified);
let seventh = &blocks_a[5];
assert_eq!(seventh.block.name(), Some("seventh"));
assert!(seventh.is_start_tag_modified);
assert!(!seventh.is_content_modified);
let eighth = &blocks_a[6];
assert_eq!(eighth.block.name(), Some("eighth"));
assert!(!eighth.is_start_tag_modified);
assert!(eighth.is_content_modified);
let ninth = &blocks_a[7];
assert_eq!(ninth.block.name(), Some("ninth"));
assert!(!ninth.is_start_tag_modified);
assert!(ninth.is_content_modified);
let tenth = &blocks_a[8];
assert_eq!(tenth.block.name(), Some("tenth"));
assert!(!tenth.is_start_tag_modified);
assert!(tenth.is_content_modified);
let blocks_b = &blocks_by_file[&RepoPath::from_reference("b.rs")?].blocks_with_context;
assert_eq!(blocks_b.len(), 1);
assert_eq!(blocks_b[0].block.name(), Some("first"));
Ok(())
}
#[test]
fn all_mode_with_line_changes_parses_modified_and_unmodified_blocks() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([
(
"a.rs".to_string(),
r#"
// <block name="first_from_a">
fn a() {}
// </block>
// <block name="second_from_a">
fn b() {
println!("hello");
}
// </block>
"#
.to_string(),
),
(
"b.rs".to_string(),
r#"
// <block name="first_from_b">
fn a() {}
// </block>
// <block name="second_from_b">
fn b() {
println!("hello");
}
// </block>
"#
.to_string(),
),
]));
let parsers = language_parsers()?;
let line_changes = HashMap::from([
(
RepoPath::from_reference("a.rs")?,
vec![LineChange {
line: 3, ranges: None,
}],
),
(
RepoPath::from_reference("b.rs")?,
vec![LineChange {
line: 3, ranges: None,
}],
),
]);
let blocks_by_file = parse_blocks(
&line_changes,
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
assert_eq!(
blocks_by_file[&RepoPath::from_reference("a.rs")?]
.blocks_with_context
.iter()
.map(|b| { (b.block.name().unwrap(), b.is_content_modified) })
.collect::<Vec<(&str, bool)>>(),
&[("first_from_a", true), ("second_from_a", false)]
);
assert_eq!(
blocks_by_file[&RepoPath::from_reference("b.rs")?]
.blocks_with_context
.iter()
.map(|b| { (b.block.name().unwrap(), b.is_content_modified) })
.collect::<Vec<(&str, bool)>>(),
&[("first_from_b", true), ("second_from_b", false)]
);
Ok(())
}
#[test]
fn all_mode_without_line_changes_parses_unmodified_blocks() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([
(
"a.rs".to_string(),
r#"
// <block name="first_from_a">
fn a() {}
// </block>
// <block name="second_from_a">
fn b() {
println!("hello");
}
// </block>
"#
.to_string(),
),
(
"b.rs".to_string(),
r#"
// <block name="first_from_b">
fn a() {}
// </block>
// <block name="second_from_b">
fn b() {
println!("hello");
}
// </block>
"#
.to_string(),
),
]));
let parsers = language_parsers()?;
let blocks_by_file = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
assert_eq!(
blocks_by_file[&RepoPath::from_reference("a.rs")?]
.blocks_with_context
.iter()
.map(|b| { (b.block.name().unwrap(), b.is_content_modified) })
.collect::<Vec<(&str, bool)>>(),
&[("first_from_a", false), ("second_from_a", false)]
);
assert_eq!(
blocks_by_file[&RepoPath::from_reference("b.rs")?]
.blocks_with_context
.iter()
.map(|b| { (b.block.name().unwrap(), b.is_content_modified) })
.collect::<Vec<(&str, bool)>>(),
&[("first_from_b", false), ("second_from_b", false)]
);
Ok(())
}
#[test]
fn parsed_blocks_contain_original_file_content() -> anyhow::Result<()> {
let file_a_contents = r#"
// <block name="first">
fn a() {}
// </block>
// <block name="second">
fn b() {
println!("hello");
println!("world");
}
// </block>
"#;
let file_system = FakeFileSystem::new(HashMap::from([(
"a.rs".to_string(),
file_a_contents.to_string(),
)]));
let parsers = language_parsers()?;
let blocks_by_file = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
let content_a = &blocks_by_file[&RepoPath::from_reference("a.rs")?].file_content;
assert_eq!(content_a, file_a_contents);
Ok(())
}
#[test]
fn with_remapped_extension_returns_parsed_blocks() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"a.rust".to_string(),
r#"
// <block name="first">
fn a() {}
// </block>"#
.to_string(),
)]));
let parsers = language_parsers()?;
let blocks_by_file = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::from([("rust".into(), "rs".into())]),
)?
.blocks;
assert_eq!(blocks_by_file.len(), 1);
assert_eq!(
blocks_by_file[&RepoPath::from_reference("a.rust")?]
.blocks_with_context
.len(),
1
);
Ok(())
}
#[test]
fn with_unknown_extension_returns_empty_result() -> anyhow::Result<()> {
let files = HashMap::from([("test.unknown".to_string(), "test content".to_string())]);
let blocks = parse_blocks(
&HashMap::new(),
ScanMode::All,
&FakeFileSystem::new(files),
&FakePathChecker::allow_all(),
&HashMap::new(),
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks.len(), 0);
Ok(())
}
#[test]
fn with_allowed_and_ignored_paths_returns_block_from_allowed_paths_only() -> anyhow::Result<()>
{
let file_system = FakeFileSystem::new(HashMap::from([
(
"allowed.rs".to_string(),
r#"
// <block name="allowed">
fn allowed() {}
// </block>
"#
.to_string(),
),
(
"ignored.rs".to_string(),
r#"
// <block name="ignored">
fn ignored() {}
// </block>
"#
.to_string(),
),
]));
let path_checker =
FakePathChecker::with_ignored_paths(HashSet::from(["ignored.rs".to_string()]));
let blocks = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&path_checker,
&language_parsers()?,
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks.len(), 1);
assert!(blocks.contains_key(&RepoPath::from_reference("allowed.rs")?));
assert!(!blocks.contains_key(&RepoPath::from_reference("ignored.rs")?));
Ok(())
}
#[test]
fn diff_target_outside_the_allowed_globs_is_not_parsed() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([
(
"src/allowed.rs".to_string(),
"// <block name=\"allowed\">\nfn allowed() {}\n// </block>\n".to_string(),
),
(
"vendor/denied.rs".to_string(),
"// <block name=\"denied\">\nfn denied() {}\n// </block>\n".to_string(),
),
]));
let line_changes = HashMap::from([
(
RepoPath::from_reference("src/allowed.rs")?,
vec![line_change(2)],
),
(
RepoPath::from_reference("vendor/denied.rs")?,
vec![line_change(2)],
),
]);
let blocks = parse_blocks(
&line_changes,
ScanMode::OnlyChanged,
&file_system,
&FakePathChecker::allow_only("src/**"),
&language_parsers()?,
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks.len(), 1);
assert!(blocks.contains_key(&RepoPath::from_reference("src/allowed.rs")?));
Ok(())
}
#[test]
fn all_mode_ignores_diff_entries_for_files_it_did_not_reach() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"present.rs".to_string(),
"// <block name=\"present\">\nfn present() {}\n// </block>\n".to_string(),
)]));
let line_changes = HashMap::from([
(
RepoPath::from_reference("present.rs")?,
vec![line_change(1)],
),
(RepoPath::from_reference("absent.rs")?, vec![line_change(1)]),
]);
let parsed = parse_blocks(
&line_changes,
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&language_parsers()?,
&HashMap::new(),
)?;
assert_eq!(parsed.blocks.len(), 1);
assert!(
parsed
.blocks
.contains_key(&RepoPath::from_reference("present.rs")?)
);
assert_eq!(parsed.stats.files_scanned, 1);
Ok(())
}
#[test]
fn diff_with_a_missing_file_reports_the_likely_cause() -> anyhow::Result<()> {
let line_changes = HashMap::from([(
RepoPath::from_reference("rules.py")?,
vec![LineChange {
line: 1,
ranges: None,
}],
)]);
let error = parse_blocks(
&line_changes,
ScanMode::OnlyChanged,
&FakeFileSystem::new(HashMap::from([("src/rules.py".to_string(), String::new())])),
&FakePathChecker::allow_all(),
&language_parsers()?,
&HashMap::new(),
)
.unwrap_err();
let message = format!("{error:#}");
assert!(
message.contains("does not exist in the repository root"),
"unexpected error: {message}"
);
Ok(())
}
#[test]
fn diff_with_only_missing_files_reports_the_likely_cause_in_all_mode() -> anyhow::Result<()> {
let line_changes =
HashMap::from([(RepoPath::from_reference("rules.py")?, vec![line_change(1)])]);
let error = parse_blocks(
&line_changes,
ScanMode::All,
&FakeFileSystem::new(HashMap::from([("src/rules.py".to_string(), String::new())])),
&FakePathChecker::allow_all(),
&language_parsers()?,
&HashMap::new(),
)
.unwrap_err();
let message = format!("{error:#}");
assert!(
message.contains("does not exist in the repository root"),
"unexpected error: {message}"
);
Ok(())
}
#[test]
fn diff_with_only_files_outside_the_globs_is_not_a_broken_diff() -> anyhow::Result<()> {
let line_changes = HashMap::from([(
RepoPath::from_reference("docs/gone.py")?,
vec![line_change(1)],
)]);
let blocks = parse_blocks(
&line_changes,
ScanMode::All,
&FakeFileSystem::new(HashMap::from([(
"src/present.py".to_string(),
"# <block name=\"present\">\nx = 1\n# </block>\n".to_string(),
)])),
&FakePathChecker::allow_only("src/**"),
&language_parsers()?,
&HashMap::new(),
)?
.blocks;
assert!(blocks.contains_key(&RepoPath::from_reference("src/present.py")?));
Ok(())
}
#[test]
fn diff_with_a_missing_ignored_file_is_skipped() -> anyhow::Result<()> {
let line_changes = HashMap::from([(
RepoPath::from_reference("vendor/gone.py")?,
vec![LineChange {
line: 1,
ranges: None,
}],
)]);
let blocks = parse_blocks(
&line_changes,
ScanMode::OnlyChanged,
&FakeFileSystem::new(HashMap::new()),
&FakePathChecker::with_ignored_paths(HashSet::from(["vendor/gone.py".to_string()])),
&language_parsers()?,
&HashMap::new(),
)?
.blocks;
assert!(blocks.is_empty());
Ok(())
}
#[test]
fn diff_with_a_missing_unparseable_file_is_skipped() -> anyhow::Result<()> {
let line_changes = HashMap::from([(
RepoPath::from_reference("assets/logo.png")?,
vec![LineChange {
line: 1,
ranges: None,
}],
)]);
let blocks = parse_blocks(
&line_changes,
ScanMode::OnlyChanged,
&FakeFileSystem::new(HashMap::new()),
&FakePathChecker::allow_all(),
&language_parsers()?,
&HashMap::new(),
)?
.blocks;
assert!(blocks.is_empty());
Ok(())
}
#[test]
fn empty_input_returns_empty_result() -> anyhow::Result<()> {
let line_changes = HashMap::default();
let blocks = parse_blocks(
&line_changes,
ScanMode::All,
&FakeFileSystem::new(HashMap::default()),
&FakePathChecker::allow_all(),
&HashMap::new(),
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks.len(), 0);
Ok(())
}
#[test]
fn parse_file_with_every_block_returns_all_blocks() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"a.py".to_string(),
"# <block name=\"x\">\n1\n# </block>\n# <block name=\"y\">\n2\n# </block>".to_string(),
)]));
let parsers = language_parsers()?;
let file_blocks = parse_file(
&file_system,
Path::new("a.py"),
&[],
every_block,
&parsers,
&HashMap::new(),
)?
.expect("python is supported");
assert_eq!(file_blocks.blocks_with_context.len(), 2);
Ok(())
}
#[test]
fn unknown_attribute_in_block_fails_with_error() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"a.py".to_string(),
"# <block name=\"x\" unknown-attr=\"value\">\n1\n# </block>".to_string(),
)]));
let parsers = language_parsers()?;
let file_blocks = parse_file(
&file_system,
Path::new("a.py"),
&[],
every_block,
&parsers,
&HashMap::new(),
);
assert!(file_blocks.is_err());
assert_eq!(
file_blocks.unwrap_err().source().unwrap().to_string(),
"Block a.py:x at line 1, column 3 contains unrecognized attribute `unknown-attr`"
);
Ok(())
}
#[test]
fn unknown_severity_value_in_block_fails_with_error() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"a.py".to_string(),
"# <block severity=\"invalid-severity\">\n1\n# </block>".to_string(),
)]));
let parsers = language_parsers()?;
let file_blocks = parse_file(
&file_system,
Path::new("a.py"),
&[],
every_block,
&parsers,
&HashMap::new(),
);
assert!(file_blocks.is_err());
assert_eq!(
file_blocks.unwrap_err().source().unwrap().to_string(),
"Block a.py:(unnamed) at line 1, column 3 contains unrecognized severity value"
);
Ok(())
}
#[test]
fn duplicate_block_name_in_the_same_file_fails_with_error() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([(
"a.py".to_string(),
"# <block name=\"x\">\n1\n# </block>\n# <block name=\"x\">\n2\n# </block>".to_string(),
)]));
let parsers = language_parsers()?;
let file_blocks = parse_file(
&file_system,
Path::new("a.py"),
&[],
every_block,
&parsers,
&HashMap::new(),
);
assert!(file_blocks.is_err());
assert_eq!(
file_blocks.unwrap_err().source().unwrap().to_string(),
"Block a.py:x at line 4, column 3 duplicates the name of the block at line 1, column 3"
);
Ok(())
}
#[test]
fn same_block_name_in_different_files_does_not_fail() -> anyhow::Result<()> {
let file_system = FakeFileSystem::new(HashMap::from([
(
"a.py".to_string(),
"# <block name=\"x\">\n1\n# </block>".to_string(),
),
(
"b.py".to_string(),
"# <block name=\"x\">\n2\n# </block>".to_string(),
),
]));
let parsers = language_parsers()?;
let blocks = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
assert_eq!(blocks.len(), 2);
Ok(())
}
}
#[cfg(test)]
mod supported_languages_tests {
use std::collections::HashMap;
use crate::blocks::*;
use crate::fs::test_utils::{FakeFileSystem, FakePathChecker};
use crate::language_parsers::language_parsers;
#[test]
fn all_language_extensions_are_supported() -> anyhow::Result<()> {
let parsers = language_parsers()?;
let files = HashMap::from([
(
"BUILD".to_string(),
"# <block>\ncc_library(name = \"foo\")\n# </block>".to_string(),
),
(
"MODULE.bazel".to_string(),
"# <block>\nmodule(name = \"m\")\n# </block>".to_string(),
),
(
"WORKSPACE".to_string(),
"# <block>\nworkspace(name = \"w\")\n# </block>".to_string(),
),
(
"WORKSPACE.bzlmod".to_string(),
"# <block>\n# migration stub\n# </block>".to_string(),
),
(
"CMakeLists.txt".to_string(),
"# <block>\nadd_library(foo foo.c)\n# </block>".to_string(),
),
(
"cmake.cmake".to_string(),
"#[[ <block> ]]\nset(X 1)\n# </block>".to_string(),
),
(
"bash.bash".to_string(),
"# <block>\necho \"hello\"\n# </block>".to_string(),
),
(
"bzl.bzl".to_string(),
"# <block>\ndef my_macro():\n pass\n# </block>".to_string(),
),
(
"c.c".to_string(),
"/* <block> */\nint main() { return 0; }\n/* </block> */".to_string(),
),
(
"cc.cpp".to_string(),
"// <block>\nint main() { return 0; }\n// </block>".to_string(),
),
(
"cpp.cpp".to_string(),
"// <block>\nint main() { return 0; }\n// </block>".to_string(),
),
(
"cs.cs".to_string(),
"// <block>\nclass Program { }\n// </block>".to_string(),
),
(
"css.css".to_string(),
"/* <block> */\nbody { margin: 0; }\n/* </block> */".to_string(),
),
(
"Containerfile".to_string(),
"# <block>\nFROM fedora\n# </block>".to_string(),
),
(
"containerfile".to_string(),
"# <block>\nFROM centos\n# </block>".to_string(),
),
(
"Dockerfile".to_string(),
"# <block>\nFROM alpine\n# </block>".to_string(),
),
(
"app.dockerfile".to_string(),
"# <block>\nFROM debian\n# </block>".to_string(),
),
(
"dart.dart".to_string(),
"// <block>\nvoid main() {}\n// </block>".to_string(),
),
(
"ex.ex".to_string(),
"# <block>\ndefmodule Foo do\nend\n# </block>".to_string(),
),
(
"exs.exs".to_string(),
"# <block>\nIO.puts(:hello)\n# </block>".to_string(),
),
(
"go.go".to_string(),
"// <block>\nfunc main() {}\n// </block>".to_string(),
),
(
"go.mod".to_string(),
"// <block>\nmodule example.com/m\n// </block>".to_string(),
),
(
"go.sum".to_string(),
"// <block>\nexample.com/dep v1.0.0 h1:abc\n// </block>".to_string(),
),
(
"go.work".to_string(),
"// <block>\nuse ./mod\n// </block>".to_string(),
),
(
"gql.gql".to_string(),
"# <block>\ntype Mutation {\n noop: Boolean\n}\n# </block>".to_string(),
),
(
"gradle.gradle".to_string(),
"// <block>\nversion = '1.0'\n// </block>".to_string(),
),
(
"graphql.graphql".to_string(),
"# <block>\ntype Query {\n hello: String\n}\n# </block>".to_string(),
),
(
"groovy.groovy".to_string(),
"// <block>\ndef x = 1\n// </block>".to_string(),
),
(
"h.h".to_string(),
"// <block>\nvoid foo();\n// </block>".to_string(),
),
(
"hcl.hcl".to_string(),
"// <block>\nregion = \"eu-west-1\"\n// </block>".to_string(),
),
(
"htm.htm".to_string(),
"<!-- <block> -->\n<div>Content</div>\n<!-- </block> -->".to_string(),
),
(
"html.html".to_string(),
"<!-- <block> -->\n<p>Hello</p>\n<!-- </block> -->".to_string(),
),
(
"java.java".to_string(),
"// <block>\nclass App {}\n// </block>".to_string(),
),
(
"Jenkinsfile".to_string(),
"// <block>\npipeline { }\n// </block>".to_string(),
),
(
"jenkinsfile".to_string(),
"// <block>\nnode { }\n// </block>".to_string(),
),
(
"js.js".to_string(),
"// <block>\nconst x = 1;\n// </block>".to_string(),
),
(
"jsx.jsx".to_string(),
"// <block>\nconst Comp = () => <div/>;\n// </block>".to_string(),
),
(
"kt.kt".to_string(),
"// <block>\nfun main() {}\n// </block>".to_string(),
),
(
"kts.kts".to_string(),
"// <block>\nplugins { }\n// </block>".to_string(),
),
(
"lua.lua".to_string(),
"-- <block>\nlocal x = 1\n-- </block>".to_string(),
),
(
"makefile".to_string(),
"# <block>\nall:\n\t@echo \"hello\"\n# </block>".to_string(),
),
(
"Makefile".to_string(),
"# <block>\nall:\n\t@echo \"hello\"\n# </block>".to_string(),
),
(
"markdown.markdown".to_string(),
"<div>\n<!-- <block> -->\n# Title\n<!-- </block> -->\n</div>".to_string(),
),
(
"md.md".to_string(),
"<div>\n<!-- <block> -->\n## Heading\n<!-- </block> -->\n</div>".to_string(),
),
(
"mk.mk".to_string(),
"# <block>\nall:\n\t@echo \"hello\"\n# </block>".to_string(),
),
(
"nix.nix".to_string(),
"# <block>\n{ pkgs = null; }\n# </block>".to_string(),
),
(
"php.php".to_string(),
"<?php\n# <block>\necho 'hello';\n# </block>\n?>".to_string(),
),
(
"phtml.phtml".to_string(),
"<?php\n# <block>\necho 'world';\n# </block>\n?>".to_string(),
),
(
"proto.proto".to_string(),
"// <block>\nsyntax = \"proto3\";\n// </block>".to_string(),
),
(
"py.py".to_string(),
"# <block>\ndef main():\n pass\n# </block>".to_string(),
),
(
"pyi.pyi".to_string(),
"# <block>\ndef foo() -> None: pass\n# </block>".to_string(),
),
(
"rb.rb".to_string(),
"# <block>\ndef hello\n puts 'world'\nend\n# </block>".to_string(),
),
(
"rs.rs".to_string(),
r#"/* <block> */fn a() {}/* </block> */"#.to_string(),
),
(
"sbt.sbt".to_string(),
"// <block>\nname := \"app\"\n// </block>".to_string(),
),
(
"scala.scala".to_string(),
"// <block>\nval x = 1\n// </block>".to_string(),
),
(
"sh.sh".to_string(),
"# <block>\necho \"hello\"\n# </block>".to_string(),
),
(
"sql.sql".to_string(),
"-- <block>\nSELECT * FROM users;\n-- </block>".to_string(),
),
(
"star.star".to_string(),
"# <block>\nx = 42\n# </block>".to_string(),
),
(
"swift.swift".to_string(),
"// <block>\nfunc main() {}\n// </block>".to_string(),
),
(
"tf.tf".to_string(),
"# <block>\nregion = \"us-east-1\"\n# </block>".to_string(),
),
(
"tfvars.tfvars".to_string(),
"# <block>\nregion = \"us-west-2\"\n# </block>".to_string(),
),
(
"toml.toml".to_string(),
"# <block>\nname = \"test\"\n# </block>".to_string(),
),
(
"ts.ts".to_string(),
"// <block>\nconst x: number = 1;\n// </block>".to_string(),
),
(
"tsx.tsx".to_string(),
"// <block>\nconst C = () => <div/>;\n// </block>".to_string(),
),
(
"typescript.d.ts".to_string(),
"// <block>\ndeclare const x: number;\n// </block>".to_string(),
),
(
"xml.xml".to_string(),
"<!-- <block> -->\n<root/>\n<!-- </block> -->".to_string(),
),
(
"yaml.yaml".to_string(),
"# <block>\nkey: value\n# </block>".to_string(),
),
(
"yml.yml".to_string(),
"# <block>\nname: test\n# </block>".to_string(),
),
]);
let file_system = FakeFileSystem::new(files.clone());
let blocks_by_file = parse_blocks(
&HashMap::new(),
ScanMode::All,
&file_system,
&FakePathChecker::allow_all(),
&parsers,
&HashMap::new(),
)?
.blocks;
for file_name in files.keys() {
assert!(
!blocks_by_file
.get(&RepoPath::from_reference(file_name)?)
.unwrap_or_else(|| panic!("No blocks found for file {file_name}"))
.blocks_with_context
.is_empty(),
"File {file_name} should have blocks",
);
}
Ok(())
}
}