use anyhow::{Context, Result};
use clap::{Arg, ArgAction, Command};
use colorful::{Color, Colorful};
use dialoguer::Confirm;
use std::fs;
use std::path::{Path, PathBuf};
use sysinfo::System;
mod config;
mod file_processor;
mod markdown_generator;
mod renderer;
mod syntax;
mod theme;
use file_processor::FileProcessor;
use markdown_generator::{FileInfo, MarkdownGenerator};
use renderer::{create_renderer, DocumentMetadata, OutputFormat};
fn format_from_extension(output_path: &Path) -> String {
match output_path
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.as_deref()
{
Some("html" | "htm") => "html",
Some("epub") => "epub",
Some("md" | "markdown") => "markdown",
_ => "pdf",
}
.to_string()
}
#[tokio::main]
async fn main() -> Result<()> {
let matches = Command::new("scrollcast")
.version(env!("CARGO_PKG_VERSION"))
.author("heartcode <0xheartcode@gmail.com>")
.about("Convert Git repositories to beautifully formatted documents")
.arg(
Arg::new("input")
.help("Input directory (git repository or regular folder)")
.required_unless_present_any(["list-themes", "list-languages", "test-project"])
.index(1)
.value_parser(clap::value_parser!(PathBuf))
)
.arg(
Arg::new("output")
.short('o')
.long("output")
.help("Output file path")
.required_unless_present_any(["list-themes", "list-languages", "test-project"])
.value_parser(clap::value_parser!(PathBuf))
)
.arg(
Arg::new("format")
.short('f')
.long("format")
.help("Output format; inferred from the output file extension when omitted (epub is experimental)")
.value_parser(["pdf", "epub", "html", "markdown"])
)
.arg(
Arg::new("include-experimental")
.long("include-experimental")
.help("Include experimental formats (EPUB) when using --test-project")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("theme")
.short('t')
.long("theme")
.help("Syntax highlighting theme")
.value_parser(["pygments", "kate", "monochrome", "breezedark", "espresso", "zenburn", "haddock", "tango"])
.default_value("kate")
)
.arg(
Arg::new("no-gitignore")
.long("no-gitignore")
.help("Ignore .gitignore files and process all files")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("no-toc")
.long("no-toc")
.help("Don't include table of contents")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("list-themes")
.long("list-themes")
.help("List available syntax highlighting themes")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("list-languages")
.long("list-languages")
.help("List supported programming languages")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("yes")
.short('y')
.long("yes")
.help("Skip confirmation prompts")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("ignore")
.long("ignore")
.help("Ignore specific directories (can be used multiple times)")
.action(ArgAction::Append)
.value_name("DIR")
)
.arg(
Arg::new("verbose")
.short('v')
.long("verbose")
.help("Enable verbose logging")
.action(ArgAction::SetTrue)
)
.arg(
Arg::new("chunk-size")
.long("chunk-size")
.help("Process files in chunks of this size to reduce memory usage")
.value_parser(clap::value_parser!(usize))
.default_value("20")
)
.arg(
Arg::new("memory-limit")
.long("memory-limit")
.help("Maximum memory usage in MB (default: 80% of available RAM)")
.value_parser(clap::value_parser!(u64))
)
.arg(
Arg::new("max-file-size")
.long("max-file-size")
.help("Maximum file size to process in MB (default: 50MB, files larger will be truncated)")
.value_parser(clap::value_parser!(u64))
.default_value("50")
)
.arg(
Arg::new("test-project")
.long("test-project")
.help("Generate test project and all output formats (cleans output_test folder)")
.action(ArgAction::SetTrue)
)
.get_matches();
if matches.get_flag("list-themes") {
list_themes()?;
return Ok(());
}
if matches.get_flag("list-languages") {
list_languages()?;
return Ok(());
}
if matches.get_flag("test-project") {
let include_experimental = matches.get_flag("include-experimental");
run_test_project(include_experimental).await?;
return Ok(());
}
let input_path = matches.get_one::<PathBuf>("input").unwrap();
let output_path = matches.get_one::<PathBuf>("output").unwrap();
let format = match matches.get_one::<String>("format") {
Some(f) => f.clone(),
None => format_from_extension(output_path),
};
let theme = matches.get_one::<String>("theme").unwrap().clone();
let respect_gitignore = !matches.get_flag("no-gitignore");
let include_toc = !matches.get_flag("no-toc");
let skip_confirmation = matches.get_flag("yes");
let verbose = matches.get_flag("verbose");
let chunk_size = *matches.get_one::<usize>("chunk-size").unwrap();
let memory_limit_mb = matches.get_one::<u64>("memory-limit").copied();
let max_file_size_mb = *matches.get_one::<u64>("max-file-size").unwrap();
let ignored_dirs: Vec<String> = matches
.get_many::<String>("ignore")
.unwrap_or_default()
.map(|s| s.to_string())
.collect();
let mut sys = System::new();
sys.refresh_memory();
let total_memory_mb = sys.total_memory() / 1024 / 1024;
let memory_limit = memory_limit_mb.unwrap_or(total_memory_mb * 80 / 100);
let output_format = match format.as_str() {
"pdf" => OutputFormat::Pdf,
"epub" => OutputFormat::Epub,
"html" => OutputFormat::Html,
"markdown" => OutputFormat::Markdown,
_ => unreachable!(), };
println!(
"{}",
"🎨 Scrollcast Document Converter".color(Color::Blue).bold()
);
println!("📂 Input: {}", input_path.display());
println!("📄 Output: {}", output_path.display());
println!("🎯 Format: {}", format.clone().color(Color::Green));
println!("🎨 Theme: {}", theme.clone().color(Color::Yellow));
println!(
"📁 Respect .gitignore: {}",
if respect_gitignore {
"Yes".color(Color::Green)
} else {
"No".color(Color::Red)
}
);
if verbose {
println!("🔍 Verbose mode: {}", "Enabled".color(Color::Green));
println!("📦 Chunk size: {} files per chunk", chunk_size);
println!(
"🧠 Memory limit: {} MB ({} MB total)",
memory_limit, total_memory_mb
);
}
if !input_path.exists() {
anyhow::bail!("Input path does not exist: {}", input_path.display());
}
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent).context("Failed to create output directory")?;
}
println!("\n{}", "📖 Processing files...".color(Color::Cyan));
let file_processor = FileProcessor::new()
.with_gitignore_respect(respect_gitignore)
.with_ignored_directories(ignored_dirs);
let files = file_processor
.process_directory(input_path)
.context("Failed to process input directory")?;
if files.is_empty() {
println!("{}", "⚠️ No files found to process".color(Color::Yellow));
return Ok(());
}
println!("✅ Found {} files to process", files.len());
if verbose {
println!("📋 Files to process:");
for (i, file) in files.iter().enumerate() {
println!(
" {}. {} ({})",
i + 1,
file.path,
format_file_size(file.size)
);
}
}
let total_size: usize = files.iter().map(|f| f.size).sum();
let avg_file_size = if !files.is_empty() {
total_size / files.len()
} else {
0
};
let large_files = files.iter().filter(|f| f.size > 50_000).count(); let huge_files = files.iter().filter(|f| f.size > 10_000_000).count(); let max_file_size = files.iter().map(|f| f.size).max().unwrap_or(0);
if huge_files > 0 {
println!(
"⚠️ Warning: Found {} files larger than 10MB. Largest file: {}",
huge_files,
format_file_size(max_file_size)
);
if max_file_size > 50_000_000 {
println!("🚨 Files over 50MB may cause memory issues. Consider using --ignore to exclude them.");
}
}
let effective_chunk_size = if files.len() > 10 || total_size > 1_000_000 || large_files > 5 {
1
} else if files.len() > 5 || avg_file_size > 10_000 {
3
} else {
chunk_size
};
let needs_chunking = files.len() > effective_chunk_size;
if needs_chunking {
if effective_chunk_size == 1 {
println!(
"📄 Processing {} files one-by-one for optimal memory usage",
files.len()
);
} else {
println!(
"📦 Processing {} files in chunks of {} to reduce memory usage",
files.len(),
effective_chunk_size
);
}
if verbose {
println!(
"📊 Repository stats: {} files, {} total, avg {} per file, {} large files (>50KB)",
files.len(),
format_file_size(total_size),
format_file_size(avg_file_size),
large_files
);
if huge_files > 0 {
println!(
"📊 Large file stats: {} files >10MB, largest: {}",
huge_files,
format_file_size(max_file_size)
);
}
}
}
if !skip_confirmation && std::io::IsTerminal::is_terminal(&std::io::stdin()) {
let proceed = Confirm::new()
.with_prompt("Do you want to proceed with processing these files?")
.default(true)
.interact()
.context("Failed to get user confirmation")?;
if !proceed {
println!("Operation cancelled by user.");
return Ok(());
}
}
println!("{}", "📝 Generating markdown...".color(Color::Cyan));
let repo_name_owned = input_path
.canonicalize()
.ok()
.as_deref()
.and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned()))
.or_else(|| {
input_path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
})
.unwrap_or_else(|| "Repository".to_string());
let repo_name = repo_name_owned.as_str();
let temp_dir = std::env::temp_dir();
let unique = format!(
"{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
);
let temp_markdown = temp_dir.join(format!("scrollcast-{repo_name}-{unique}.tmp.md"));
if needs_chunking {
process_files_in_chunks(
&files,
repo_name,
effective_chunk_size,
&temp_markdown,
include_toc,
verbose,
memory_limit,
max_file_size_mb,
)
.await
.context("Failed to process files in chunks")?;
} else {
let markdown_generator = MarkdownGenerator::new(include_toc, true);
let markdown_content = markdown_generator
.generate_markdown(&files, repo_name)
.context("Failed to generate markdown")?;
fs::write(&temp_markdown, &markdown_content)
.context("Failed to write temporary markdown file")?;
}
println!("✅ Markdown generated");
if verbose {
let markdown_size = fs::metadata(&temp_markdown)?.len();
println!("📄 Markdown file size: {} bytes", markdown_size);
println!("📂 Temporary markdown file: {}", temp_markdown.display());
}
println!("{}", "🔄 Converting to final format...".color(Color::Cyan));
if !matches!(output_format, OutputFormat::Markdown) {
let metadata = DocumentMetadata {
title: repo_name.to_string(),
author: None,
date: Some(chrono::Utc::now().format("%Y-%m-%d").to_string()),
language: "en".to_string(),
syntax_theme: theme.clone(),
};
let renderer = create_renderer(&output_format).context("Failed to create renderer")?;
let markdown_content =
fs::read_to_string(&temp_markdown).context("Failed to read temporary markdown file")?;
renderer
.save_to_file(&markdown_content, &metadata, output_path)
.context("Failed to render document")?;
} else {
fs::copy(&temp_markdown, output_path).context("Failed to copy markdown file")?;
}
let _ = fs::remove_file(&temp_markdown);
println!(
"\n{} Document generated successfully!",
"🎉".color(Color::Green)
);
println!(
"📄 Output: {}",
output_path.display().to_string().color(Color::Blue)
);
if let Ok(metadata) = fs::metadata(output_path) {
let size = metadata.len();
let size_str = if size > 1_048_576 {
format!("{:.1} MB", size as f64 / 1_048_576.0)
} else if size > 1024 {
format!("{:.1} KB", size as f64 / 1024.0)
} else {
format!("{} bytes", size)
};
println!("📊 File size: {}", size_str.color(Color::Green));
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn process_files_in_chunks(
files: &[FileInfo],
repo_name: &str,
chunk_size: usize,
output_path: &Path,
include_toc: bool,
verbose: bool,
memory_limit_mb: u64,
max_file_size_mb: u64,
) -> Result<()> {
let mut sys = System::new();
let mut final_markdown = String::new();
final_markdown.push_str(&format!("# {}\n\n", repo_name));
final_markdown.push_str(&format!(
"Generated on: {}\n\n",
chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC")
));
if include_toc {
final_markdown.push_str("## Table of Contents\n\n");
for file in files {
let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
let escaped_path = escape_markdown_special_chars(&file.path);
final_markdown.push_str(&format!("- [{}](#{sanitized_path})\n", escaped_path));
}
final_markdown.push('\n');
}
final_markdown.push_str("## File Structure\n\n");
final_markdown.push_str("```\n");
for file in files {
final_markdown.push_str(&format!("{}\n", file.path));
}
final_markdown.push_str("```\n\n");
final_markdown.push_str("## File Contents\n\n");
let chunks: Vec<&[FileInfo]> = files.chunks(chunk_size).collect();
let total_chunks = chunks.len();
let mut _global_page_number = 1; let mut file_counter = 0;
for (chunk_index, chunk) in chunks.iter().enumerate() {
if verbose {
println!(
"📄 Processing chunk {} of {} ({} files)",
chunk_index + 1,
total_chunks,
chunk.len()
);
}
for file in chunk.iter() {
file_counter += 1;
_global_page_number += 1;
if verbose {
sys.refresh_memory();
let used_memory_mb = sys.used_memory() / 1024 / 1024;
let file_size_str = if file.size > 10_000_000 {
format!("{} ⚠️", format_file_size(file.size))
} else {
format_file_size(file.size)
};
println!(
" 📄 Processing file {}/{}: {} ({}) [Memory: {} MB/{} MB]",
file_counter,
files.len(),
file.path,
file_size_str,
used_memory_mb,
memory_limit_mb
);
if used_memory_mb > memory_limit_mb {
println!(
"⚠️ Warning: Memory usage ({} MB) exceeds limit ({} MB)",
used_memory_mb, memory_limit_mb
);
}
}
let max_file_size_bytes = max_file_size_mb * 1024 * 1024;
let processed_content = if file.size > max_file_size_bytes as usize {
if verbose {
println!(
" 🔄 File too large ({}), showing first 100KB + summary",
format_file_size(file.size)
);
}
truncate_large_file_content(&file.content, file.size)
} else {
file.content.clone()
};
final_markdown.push_str("\n\\newpage\n\n");
let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
let escaped_path = escape_markdown_special_chars(&file.path);
final_markdown.push_str(&format!("### {} {{#{sanitized_path}}}\n\n", escaped_path));
final_markdown.push_str(&format!("**Size:** {}\n\n", format_file_size(file.size)));
if file.path.ends_with(".md") || file.path.ends_with(".markdown") {
if verbose {
println!(
" 📝 Processing markdown file: {} (no code block)",
file.path
);
}
final_markdown.push_str(&processed_content);
if !processed_content.ends_with('\n') {
final_markdown.push('\n');
}
} else {
if verbose {
println!(
" 💻 Processing code file: {} (with code block)",
file.path
);
}
if let Some(language) = &file.language {
final_markdown.push_str(&format!("```{}\n", language));
} else {
final_markdown.push_str("```\n");
}
final_markdown.push_str(&processed_content);
if !processed_content.ends_with('\n') {
final_markdown.push('\n');
}
final_markdown.push_str("```\n\n");
}
final_markdown.push_str("---\n\n");
}
if chunk_index < total_chunks - 1 {
tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
}
}
fs::write(output_path, final_markdown).context("Failed to write chunked markdown file")?;
Ok(())
}
fn escape_markdown_special_chars(text: &str) -> String {
text.replace('_', "\\_") .replace('#', "\\#") .replace('$', "\\$") .replace('%', "\\%") .replace('&', "\\&") .replace('^', "\\^") .replace('{', "\\{") .replace('}', "\\}")
}
fn format_file_size(size: usize) -> String {
const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"];
let mut size_f = size as f64;
let mut unit_index = 0;
while size_f >= 1024.0 && unit_index < UNITS.len() - 1 {
size_f /= 1024.0;
unit_index += 1;
}
if unit_index == 0 {
format!("{} {}", size, UNITS[unit_index])
} else {
format!("{:.1} {}", size_f, UNITS[unit_index])
}
}
fn truncate_large_file_content(content: &str, original_size: usize) -> String {
const PREVIEW_SIZE: usize = 100_000; const SAMPLE_SIZE: usize = 10_000; const MAX_SAMPLES: usize = 5;
if content.len() <= PREVIEW_SIZE {
return content.to_string();
}
let mut result = String::new();
let preview_end = std::cmp::min(PREVIEW_SIZE, content.len());
result.push_str(&content[..preview_end]);
result.push_str("\n\n");
result.push_str(&format!(
"... [File continues for {} more] ...\n\n",
format_file_size(original_size - preview_end)
));
let remaining = content.len() - preview_end;
if remaining > SAMPLE_SIZE * 2 {
for i in 1..=MAX_SAMPLES {
let sample_start = preview_end + (remaining * i) / (MAX_SAMPLES + 1);
let sample_end = std::cmp::min(sample_start + SAMPLE_SIZE, content.len());
if sample_start < content.len() {
result.push_str(&format!(
"\n--- Sample {} (around {}%) ---\n",
i,
(sample_start * 100) / content.len()
));
result.push_str(&content[sample_start..sample_end]);
result.push('\n');
}
}
}
result.push_str("\n\n--- File Summary ---\n");
result.push_str(&format!(
"Total size: {}\n",
format_file_size(original_size)
));
result.push_str(&format!(
"Lines shown: ~{} of ~{}\n",
result.lines().count(),
content.lines().count()
));
result.push_str("Note: Large file truncated to prevent memory issues.\n");
result
}
fn list_themes() -> Result<()> {
println!(
"{}",
"Available syntax highlighting themes:"
.color(Color::Blue)
.bold()
);
println!(" • {}", "pygments".color(Color::Green));
println!(" • {}", "kate".color(Color::Green));
println!(" • {}", "monochrome".color(Color::Green));
println!(" • {}", "breezedark".color(Color::Green));
println!(" • {}", "espresso".color(Color::Green));
println!(" • {}", "zenburn".color(Color::Green));
println!(" • {}", "haddock".color(Color::Green));
println!(" • {}", "tango".color(Color::Green));
Ok(())
}
fn list_languages() -> Result<()> {
println!(
"{}",
"Supported programming languages:".color(Color::Blue).bold()
);
let languages = vec![
"Rust",
"Python",
"JavaScript",
"TypeScript",
"Go",
"Java",
"C",
"C++",
"C#",
"Ruby",
"PHP",
"Swift",
"Kotlin",
"Scala",
"Haskell",
"Lua",
"R",
"MATLAB",
"Julia",
"Perl",
"Shell",
"PowerShell",
"SQL",
"HTML",
"CSS",
"XML",
"JSON",
"YAML",
"TOML",
"Markdown",
"LaTeX",
"Dockerfile",
"Makefile",
"CMake",
"Nginx",
"Apache",
];
let mut count = 0;
for language in languages {
print!("{:<20}", language);
count += 1;
if count % 4 == 0 {
println!();
}
}
if count % 4 != 0 {
println!();
}
println!(
"\n{}",
"Note: Syntect supports 100+ languages with TextMate syntax definitions."
.color(Color::Yellow)
);
Ok(())
}
async fn run_test_project(include_experimental: bool) -> Result<()> {
use std::process::Command;
println!(
"{}",
"🧪 Running Test Project Generation"
.color(Color::Blue)
.bold()
);
if include_experimental {
println!(
"{}",
"🧪 Including experimental formats (EPUB)".color(Color::Yellow)
);
} else {
println!(
"{}",
"📝 Skipping experimental formats (use --include-experimental to include EPUB)"
.color(Color::Yellow)
);
}
println!(
"{}",
"🧹 Cleaning testfiles/output_test folder...".color(Color::Cyan)
);
if Path::new("testfiles/output_test").exists() {
fs::remove_dir_all("testfiles/output_test")
.context("Failed to remove testfiles/output_test directory")?;
}
fs::create_dir_all("testfiles/output_test")
.context("Failed to create testfiles/output_test directory")?;
println!(
"{}",
"📁 Checking testfiles/test_project...".color(Color::Cyan)
);
if !Path::new("testfiles/test_project").exists() {
println!("❌ testfiles/test_project directory not found!");
println!("Please create a testfiles/test_project directory with some files to test with.");
return Ok(());
}
let test_files = fs::read_dir("testfiles/test_project")
.context("Failed to read testfiles/test_project directory")?
.count();
if test_files == 0 {
println!("❌ testfiles/test_project directory is empty!");
println!("Please add some files to the testfiles/test_project directory to test with.");
return Ok(());
}
println!(
"✅ Found testfiles/test_project directory with {} files",
test_files
);
let formats = if include_experimental {
vec!["markdown", "html", "epub", "pdf"]
} else {
vec!["markdown", "html", "pdf"]
};
let mut success_count = 0;
let mut failed_formats = Vec::new();
for format in &formats {
println!(
"{}",
format!("📄 Generating {} format...", format).color(Color::Cyan)
);
let output_file = format!(
"testfiles/output_test/test_project.{}",
match *format {
"markdown" => "md",
other => other,
}
);
let result = Command::new("cargo")
.args([
"run",
"--",
"testfiles/test_project",
"--output",
&output_file,
"--format",
format,
"--yes",
])
.status();
match result {
Ok(status) if status.success() => {
println!("✅ {} generated successfully", format);
success_count += 1;
if let Ok(metadata) = fs::metadata(&output_file) {
let size = metadata.len();
let size_str = if size > 1_048_576 {
format!("{:.1} MB", size as f64 / 1_048_576.0)
} else if size > 1024 {
format!("{:.1} KB", size as f64 / 1024.0)
} else {
format!("{} bytes", size)
};
println!(" 📊 File size: {}", size_str.color(Color::Green));
}
}
Ok(_) => {
println!("❌ Failed to generate {}", format);
failed_formats.push(format);
}
Err(e) => {
println!("❌ Error generating {}: {}", format, e);
failed_formats.push(format);
}
}
}
println!(
"\n{}",
"📊 Test Project Generation Summary"
.color(Color::Blue)
.bold()
);
println!(
"✅ Successfully generated: {}/{} formats",
success_count,
formats.len()
);
if !failed_formats.is_empty() {
let failed_list: Vec<String> = failed_formats.iter().map(|f| f.to_string()).collect();
println!("❌ Failed formats: {}", failed_list.join(", "));
}
println!(
"📁 Output directory: {}",
"testfiles/output_test/".color(Color::Blue)
);
if let Ok(entries) = fs::read_dir("testfiles/output_test") {
println!("\n{}", "📄 Generated files:".color(Color::Cyan));
for entry in entries.flatten() {
let path = entry.path();
if path.is_file() {
let filename = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
if let Ok(metadata) = entry.metadata() {
let size = metadata.len();
let size_str = if size > 1_048_576 {
format!("{:.1} MB", size as f64 / 1_048_576.0)
} else if size > 1024 {
format!("{:.1} KB", size as f64 / 1024.0)
} else {
format!("{} bytes", size)
};
println!(" 📄 {} ({})", filename.color(Color::Green), size_str);
} else {
println!(" 📄 {}", filename.color(Color::Green));
}
}
}
}
Ok(())
}