use std::collections::BTreeMap;
use std::io::{Read as _, Seek as _, SeekFrom};
use std::path::{Path, PathBuf};
use crate::output::human_bytes;
pub const FEED_WINDOW_BYTES: u64 = 64 * 1024;
pub const FEED_TAIL_LINES: usize = 40;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Stream {
Out,
Err,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TailLine {
pub stream: Stream,
pub text: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Tail {
pub lines: Vec<TailLine>,
pub missed_lines: usize,
pub missed_bytes: u64,
pub read_bytes: u64,
pub note: Option<String>,
}
pub fn read(seen: &mut BTreeMap<PathBuf, u64>, out: Option<&Path>, err: Option<&Path>) -> Tail {
let mut tail = Tail::default();
let mut notes: Vec<String> = Vec::new();
if out.is_none() && err.is_none() {
tail.note = Some("the shepherd did not report a log path for this sheep".to_string());
return tail;
}
for (stream, path) in [(Stream::Out, out), (Stream::Err, err)] {
let Some(path) = path else { continue };
match read_window(seen, path) {
Ok(window) => {
tail.read_bytes = tail.read_bytes.saturating_add(window.read_bytes);
tail.missed_bytes = tail.missed_bytes.saturating_add(window.never_read);
tail.missed_lines = tail.missed_lines.saturating_add(window.dropped);
tail.lines.extend(
window
.lines
.into_iter()
.map(|text| TailLine { stream, text }),
);
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => notes.push(format!(
"this sheep has not written a log in this $SHEP_HOME ({})",
path.display()
)),
Err(err) => notes.push(format!("could not read {}: {err}", path.display())),
}
}
if tail.lines.is_empty() {
tail.note = Some(if !notes.is_empty() {
notes.join("; ")
} else if tail.read_bytes > 0 {
format!(
"this sheep's last {} of log contains no complete line",
human_bytes(FEED_WINDOW_BYTES)
)
} else {
"this sheep has written nothing yet".to_string()
});
}
tail
}
struct Window {
lines: Vec<String>,
dropped: usize,
never_read: u64,
read_bytes: u64,
}
fn read_window(seen: &mut BTreeMap<PathBuf, u64>, path: &Path) -> std::io::Result<Window> {
let mut file = std::fs::File::open(path)?;
let len = file.metadata()?.len();
let start = len.saturating_sub(FEED_WINDOW_BYTES);
if start > 0 {
file.seek(SeekFrom::Start(start))?;
}
let mut window = Vec::with_capacity(usize::try_from(len.min(FEED_WINDOW_BYTES)).unwrap_or(0));
(&mut file)
.take(FEED_WINDOW_BYTES)
.read_to_end(&mut window)?;
let read_bytes = u64::try_from(window.len()).unwrap_or(u64::MAX);
let mut dropped = usize::from(start > 0);
let bytes: &[u8] = if start > 0 {
match window.iter().position(|&byte| byte == b'\n') {
Some(newline) => &window[newline + 1..],
None => &[],
}
} else {
&window
};
let text = String::from_utf8_lossy(bytes);
let mut lines: Vec<String> = text.split('\n').map(String::from).collect();
if lines.last().is_some_and(String::is_empty) {
lines.pop();
}
let keep_from = lines.len().saturating_sub(FEED_TAIL_LINES);
dropped += keep_from;
lines.drain(..keep_from);
let previous = seen.insert(path.to_path_buf(), len);
let never_read = match previous {
None => 0,
Some(previous) => start.saturating_sub(previous),
};
Ok(Window {
lines,
dropped,
never_read,
read_bytes,
})
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::io::Write as _;
use super::*;
#[test]
fn a_four_megabyte_file_costs_one_window_and_forty_lines() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
let line = "x".repeat(120);
let mut body = String::new();
while body.len() < 4 * 1024 * 1024 {
body.push_str(&line);
body.push('\n');
}
std::fs::write(&path, &body).unwrap();
let mut seen = BTreeMap::new();
let tail = read(&mut seen, Some(&path), None);
assert!(
tail.read_bytes <= FEED_WINDOW_BYTES,
"the reader pulled {} bytes off a 4 MiB file",
tail.read_bytes
);
assert_eq!(tail.lines.len(), FEED_TAIL_LINES);
assert_eq!(
tail.missed_bytes, 0,
"the first read of a file is not a gap BETWEEN READS"
);
assert!(
tail.missed_lines > 400,
"a 64 KiB window of 121-byte lines holds ~540 of them and keeps 40; \
counted only {}",
tail.missed_lines
);
}
#[test]
fn the_lines_the_cap_dropped_are_counted_even_when_no_bytes_were_skipped() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
let sixty: String = (0..60).map(|n| format!("line-{n}\n")).collect();
assert!(
sixty.len() < usize::try_from(FEED_WINDOW_BYTES).unwrap(),
"the fixture has to sit well inside one window or it tests the wrong thing"
);
std::fs::write(&path, &sixty).unwrap();
let mut seen = BTreeMap::new();
let tail = read(&mut seen, Some(&path), None);
assert_eq!(tail.missed_bytes, 0, "nothing overran the window");
assert_eq!(tail.lines.len(), FEED_TAIL_LINES);
assert_eq!(tail.missed_lines, 20, "sixty in, forty kept");
assert_eq!(tail.lines[0].text, "line-20", "and it is the NEWEST forty");
}
#[test]
fn a_file_that_grows_during_the_read_is_still_bounded_by_the_reader() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
std::fs::write(&path, "seed\n".repeat(200_000)).unwrap();
let writing = path.clone();
let writer = std::thread::spawn(move || {
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&writing)
.unwrap();
let chunk = "w".repeat(64 * 1024 - 1);
for _ in 0..512 {
writeln!(file, "{chunk}").unwrap();
}
});
let mut seen = BTreeMap::new();
let mut worst = 0;
for _ in 0..200 {
worst = worst.max(read(&mut seen, Some(&path), None).read_bytes);
}
writer.join().unwrap();
assert!(
worst <= FEED_WINDOW_BYTES,
"one read pulled {worst} bytes off a file that was still being written"
);
assert!(
std::fs::metadata(&path).unwrap().len() > 32 * 1024 * 1024,
"the writer did not get far enough for this test to mean anything"
);
}
#[test]
fn a_file_that_grew_between_reads_reports_the_bytes_it_skipped() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
std::fs::write(&path, "one\ntwo\n").unwrap();
let mut seen = BTreeMap::new();
let first = read(&mut seen, Some(&path), None);
assert_eq!(first.missed_bytes, 0);
assert_eq!(first.lines.len(), 2);
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap();
let burst = "y".repeat(4 * 1024 * 1024);
writeln!(file, "{burst}").unwrap();
writeln!(file, "three").unwrap();
writeln!(file, "four").unwrap();
drop(file);
let second = read(&mut seen, Some(&path), None);
assert!(
second.missed_bytes > 4 * 1024 * 1024 - 2 * FEED_WINDOW_BYTES,
"got {}",
second.missed_bytes
);
assert!(
second.missed_bytes < 4 * 1024 * 1024,
"the last window WAS read, so it does not belong in the gap: {}",
second.missed_bytes
);
assert_eq!(
second.lines.last().unwrap().text,
"four",
"the NEWEST lines survive"
);
assert_eq!(
second.missed_lines, 1,
"the four-megabyte line the window cut in half is one line, counted"
);
let third = read(&mut seen, Some(&path), None);
assert_eq!(third.missed_bytes, 0);
assert_eq!(third.missed_lines, 1);
}
#[test]
fn a_truncated_file_reports_no_gap_and_re_reads_from_the_top() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
std::fs::write(&path, "a\nb\nc\nd\n").unwrap();
let mut seen = BTreeMap::new();
let _ = read(&mut seen, Some(&path), None);
std::fs::write(&path, "fresh\n").unwrap();
let after = read(&mut seen, Some(&path), None);
assert_eq!(after.missed_bytes, 0);
assert_eq!(after.missed_lines, 0, "and nothing was dropped either");
assert_eq!(after.lines.len(), 1);
assert_eq!(after.lines[0].text, "fresh");
}
#[test]
fn a_window_boundary_discards_the_partial_line_it_lands_in() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("web-out.log");
let filler = "z".repeat(usize::try_from(FEED_WINDOW_BYTES).unwrap());
std::fs::write(&path, format!("{filler}PARTIAL-HEAD\nwhole-line\n")).unwrap();
let mut seen = BTreeMap::new();
let tail = read(&mut seen, Some(&path), None);
assert!(
!tail.lines.iter().any(|l| l.text.contains("PARTIAL")),
"a line cut by the window must be dropped, not shown: {:?}",
tail.lines
);
assert_eq!(tail.lines.last().unwrap().text, "whole-line");
assert_eq!(
tail.missed_lines, 1,
"dropped is not the same as hidden: the cut line is counted"
);
}
#[test]
fn both_streams_are_tagged_and_stderr_comes_last() {
let dir = tempfile::tempdir().unwrap();
let out = dir.path().join("web-out.log");
let err = dir.path().join("web-err.log");
std::fs::write(&out, "hello\n").unwrap();
std::fs::write(&err, "panicked at 'boom'\n").unwrap();
let mut seen = BTreeMap::new();
let tail = read(&mut seen, Some(&out), Some(&err));
assert_eq!(
tail.lines[0],
TailLine {
stream: Stream::Out,
text: "hello".to_string()
}
);
assert_eq!(
tail.lines.last().unwrap(),
&TailLine {
stream: Stream::Err,
text: "panicked at 'boom'".to_string()
}
);
assert_eq!(tail.note, None, "there was something to show");
}
#[test]
fn each_reason_the_feed_is_empty_gets_its_own_sentence() {
let dir = tempfile::tempdir().unwrap();
let mut seen = BTreeMap::new();
let unknown = read(&mut seen, None, None);
assert!(unknown.lines.is_empty());
assert!(
unknown
.note
.as_deref()
.unwrap()
.contains("did not report a log path"),
"got {:?}",
unknown.note
);
let missing = read(&mut seen, Some(&dir.path().join("nope.log")), None);
assert!(
missing
.note
.as_deref()
.unwrap()
.contains("has not written a log"),
"got {:?}",
missing.note
);
let as_dir = dir.path().join("a-directory.log");
std::fs::create_dir(&as_dir).unwrap();
let unreadable = read(&mut seen, Some(&as_dir), None);
assert!(
unreadable
.note
.as_deref()
.unwrap()
.contains("could not read"),
"got {:?}",
unreadable.note
);
let quiet = dir.path().join("quiet.log");
std::fs::write(&quiet, "").unwrap();
let silent = read(&mut seen, Some(&quiet), None);
assert!(silent.lines.is_empty());
assert!(
silent
.note
.as_deref()
.unwrap()
.contains("has written nothing"),
"got {:?}",
silent.note
);
let unterminated = dir.path().join("one-long-line.log");
std::fs::write(
&unterminated,
"q".repeat(usize::try_from(FEED_WINDOW_BYTES).unwrap() + 10),
)
.unwrap();
let long = read(&mut seen, Some(&unterminated), None);
assert!(long.lines.is_empty());
assert!(long.read_bytes > 0, "it read plenty; it just found no line");
assert!(
long.note.as_deref().unwrap().contains("no complete line"),
"got {:?}",
long.note
);
}
}