rust-a-log 0.1.1

A fast file log viewer and TUI for local and LAN application streams
Documentation
use std::borrow::Cow;
use std::fs::{self, File, Metadata};
use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom};
use std::path::Path;
use std::thread;
use std::time::Duration;

const BLOCK_SIZE: usize = 64 * 1024;

fn text(bytes: &[u8]) -> Cow<'_, str> {
    let bytes = bytes
        .strip_suffix(b"\n")
        .map(|line| line.strip_suffix(b"\r").unwrap_or(line))
        .unwrap_or(bytes);
    String::from_utf8_lossy(bytes)
}

/// Unfiltered tails need only newline positions, not decoded/copied records.
pub fn find_plain_start(
    reader: &mut (impl Read + Seek),
    end: u64,
    count: usize,
) -> io::Result<u64> {
    if count == 0 {
        return Ok(end);
    }
    let mut position = end;
    let mut block = [0; BLOCK_SIZE];
    let mut found = 0;
    while position > 0 {
        let size = position.min(BLOCK_SIZE as u64) as usize;
        position -= size as u64;
        reader.seek(SeekFrom::Start(position))?;
        reader.read_exact(&mut block[..size])?;
        for (index, &byte) in block[..size].iter().enumerate().rev() {
            let offset = position + index as u64;
            if byte == b'\n' && offset + 1 != end {
                found += 1;
                if found == count {
                    return Ok(offset + 1);
                }
            }
        }
    }
    Ok(0)
}

/// Scan backwards until enough matching records are found, keeping one block
/// and one record in memory. The caller supplies a fixed EOF for a growing file.
pub fn find_start(
    reader: &mut (impl Read + Seek),
    end: u64,
    count: usize,
    matches: impl Fn(&str) -> bool,
) -> io::Result<u64> {
    if count == 0 {
        return Ok(end);
    }
    let mut position = end;
    let mut block = [0; BLOCK_SIZE];
    let mut reversed = Vec::new();
    let mut found = 0;
    while position > 0 {
        let size = position.min(BLOCK_SIZE as u64) as usize;
        position -= size as u64;
        reader.seek(SeekFrom::Start(position))?;
        reader.read_exact(&mut block[..size])?;
        for (index, &byte) in block[..size].iter().enumerate().rev() {
            let offset = position + index as u64;
            if byte == b'\n' && offset + 1 != end {
                reversed.reverse();
                if matches(&text(&reversed)) {
                    found += 1;
                    if found == count {
                        return Ok(offset + 1);
                    }
                }
                reversed.clear();
            }
            reversed.push(byte);
        }
    }
    Ok(0)
}

pub fn read_records(
    reader: &mut impl BufRead,
    pending: &mut Vec<u8>,
    emit: impl FnMut(&str) -> io::Result<()>,
) -> io::Result<()> {
    read_records_bounded(reader, pending, emit, usize::MAX)
}

pub fn read_records_bounded(
    reader: &mut impl BufRead,
    pending: &mut Vec<u8>,
    mut emit: impl FnMut(&str) -> io::Result<()>,
    limit: usize,
) -> io::Result<()> {
    loop {
        let bytes = if limit == usize::MAX {
            reader.read_until(b'\n', pending)?
        } else {
            Read::by_ref(reader)
                .take((limit + 1 - pending.len()) as u64)
                .read_until(b'\n', pending)?
        };
        if pending.len() > limit {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                "live file record exceeds 64 KiB",
            ));
        }
        if bytes == 0 {
            break;
        }
        if pending.ends_with(b"\n") {
            emit(&text(pending))?;
            pending.clear();
        }
    }
    Ok(())
}

fn replaced(previous: &Metadata, current: &Metadata) -> bool {
    #[cfg(unix)]
    {
        use std::os::unix::fs::MetadataExt;
        (previous.dev(), previous.ino()) != (current.dev(), current.ino())
    }
    #[cfg(not(unix))]
    {
        let _ = (previous, current);
        false
    }
}

pub fn follow(
    path: &Path,
    file: File,
    pending: Vec<u8>,
    emit: impl FnMut(&str) -> io::Result<()>,
) -> io::Result<()> {
    follow_until(path, file, pending, emit, || true, usize::MAX)
}

pub fn follow_until(
    path: &Path,
    file: File,
    mut pending: Vec<u8>,
    mut emit: impl FnMut(&str) -> io::Result<()>,
    running: impl Fn() -> bool,
    limit: usize,
) -> io::Result<()> {
    let mut reader = BufReader::new(file);
    while running() {
        let metadata = reader.get_ref().metadata()?;
        if metadata.len() < reader.stream_position()? {
            reader.seek(SeekFrom::Start(0))?;
            pending.clear();
        }
        read_records_bounded(&mut reader, &mut pending, &mut emit, limit)?;
        match fs::metadata(path) {
            Ok(current) if replaced(&metadata, &current) => match File::open(path) {
                Ok(file) => {
                    if !file.metadata()?.is_file() {
                        return Err(io::Error::new(
                            io::ErrorKind::InvalidInput,
                            "expected a regular log file",
                        ));
                    }
                    reader = BufReader::new(file);
                    pending.clear();
                    continue;
                }
                Err(error) if error.kind() == io::ErrorKind::NotFound => {}
                Err(error) => return Err(error),
            },
            Ok(_) => {}
            Err(error) if error.kind() == io::ErrorKind::NotFound => {}
            Err(error) => return Err(error),
        }
        thread::sleep(Duration::from_millis(200));
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Cursor;

    #[test]
    fn backwards_scan_matches_forward_reference_at_block_boundaries() {
        let mut cases = vec![
            String::new(),
            "\n".into(),
            "\n\n".into(),
            "first\nlast".into(),
            "first\r\nlast\r\n".into(),
            "last\r".into(),
        ];
        for length in [BLOCK_SIZE - 1, BLOCK_SIZE, BLOCK_SIZE + 1, BLOCK_SIZE * 3] {
            cases.push(format!("first\n{}\nmatch\nlast", "ž".repeat(length)));
        }
        for input in cases {
            for count in [0, 1, 2, 10, usize::MAX] {
                for filtered in [false, true] {
                    let matches = |line: &str| !filtered || line.contains("match");
                    let all: Vec<_> = input.lines().filter(|line| matches(line)).collect();
                    let expected = &all[all.len().saturating_sub(count)..];
                    let start = find_start(
                        &mut Cursor::new(input.as_bytes()),
                        input.len() as u64,
                        count,
                        matches,
                    )
                    .unwrap();
                    let actual: Vec<_> = input[start as usize..]
                        .lines()
                        .filter(|line| matches(line))
                        .collect();
                    assert_eq!(
                        actual,
                        expected,
                        "length={}, count={count}, filtered={filtered}",
                        input.len()
                    );
                }
            }
        }
    }

    #[test]
    fn recent_records_never_read_the_large_prefix() {
        struct GuardedFile(Cursor<Vec<u8>>);
        impl Read for GuardedFile {
            fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
                assert!(
                    self.0.position() >= self.0.get_ref().len() as u64 - BLOCK_SIZE as u64,
                    "reading old data to retrieve recent records"
                );
                self.0.read(buffer)
            }
        }
        impl Seek for GuardedFile {
            fn seek(&mut self, position: SeekFrom) -> io::Result<u64> {
                self.0.seek(position)
            }
        }
        let mut bytes = vec![b'x'; BLOCK_SIZE * 4];
        bytes.extend_from_slice(b"\n[ERROR] old\n[INFO] last\n");
        let end = bytes.len() as u64;
        let mut reader = GuardedFile(Cursor::new(bytes));
        assert_eq!(find_start(&mut reader, end, 1, |_| true).unwrap(), end - 12);
        assert_eq!(find_start(&mut reader, end, 0, |_| true).unwrap(), end);
        assert_eq!(find_plain_start(&mut reader, end, 1).unwrap(), end - 12);
    }

    #[test]
    fn incomplete_utf8_records_are_emitted_only_once_when_completed() {
        let mut pending = Vec::new();
        let mut lines = Vec::new();
        for bytes in [
            &b"[INFO] \xc5"[..],
            &b"\xbe partial"[..],
            &b"\r\nnext\n"[..],
        ] {
            read_records(&mut Cursor::new(bytes), &mut pending, |line| {
                lines.push(line.to_owned());
                Ok(())
            })
            .unwrap();
        }
        assert_eq!(lines, ["[INFO] ž partial", "next"]);
        assert!(pending.is_empty());
    }

    #[test]
    fn unfiltered_tail_matches_lines_without_decoding_large_records() {
        for input in ["", "\n", "\n\n", "a\nb", "a\nb\n", "a\r\nb\r\n", "ž\n\nz"] {
            for count in [0, 1, 2, 10] {
                let all: Vec<_> = input.lines().collect();
                let start = find_plain_start(
                    &mut Cursor::new(input.as_bytes()),
                    input.len() as u64,
                    count,
                )
                .unwrap();
                assert_eq!(
                    input[start as usize..].lines().collect::<Vec<_>>(),
                    all[all.len().saturating_sub(count)..]
                );
            }
        }
        let input = format!("{}\nlast\n", "x".repeat(BLOCK_SIZE * 4));
        let start =
            find_plain_start(&mut Cursor::new(input.as_bytes()), input.len() as u64, 1).unwrap();
        assert_eq!(&input[start as usize..], "last\n");
    }

    #[test]
    fn live_file_reader_limits_incomplete_records_without_limiting_legacy_files() {
        let bytes = vec![b'x'; 70_000];
        let mut pending = Vec::new();
        assert!(
            read_records_bounded(&mut Cursor::new(&bytes), &mut pending, |_| Ok(()), 65_536)
                .is_err()
        );
        assert_eq!(pending.len(), 65_537);
        pending.clear();
        read_records(&mut Cursor::new(&bytes), &mut pending, |_| Ok(())).unwrap();
        assert_eq!(pending.len(), bytes.len());
    }
}