depbank 0.2.7

A Rust CLI tool for generating AI-friendly code banks from dependencies. Automatically parses Cargo.toml files, resolves versions, and generates searchable documentation while calculating token counts.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
use anyhow::{Context, Result};
use depbank::{
    DependencyCollection, calculate_directory_tokens, calculate_file_tokens, collect_dependencies,
    extract_dependency_info, find_cargo_lock, find_cargo_toml_files, generate_all_code_banks,
    is_dependency_available, resolve_dependency_versions, resolve_registry_path,
};
use std::collections::{HashMap, HashSet};
use std::fmt::Write as _;
use std::fs;
use std::path::{Path, PathBuf};

// Constants for formatting strings
const README_HEADER: &str = "# Code Bank Summary\n\n";
const README_TABLE_HEADER: &str = "| Dependency | Version | Tokens | Size (bytes) |\n";
const README_TABLE_SEPARATOR: &str = "|------------|---------|--------|-------------|\n";
const README_TOKEN_SUMMARY_HEADER: &str = "\n## Token Summary\n\n";
const README_ABOUT_HEADER: &str = "\n## About Code Banks\n\n";
const README_ABOUT_P1: &str = "Code banks are generated summaries of your project's dependencies. ";
const README_ABOUT_P2: &str = "They provide an overview of the code structure and key components ";
const README_ABOUT_P3: &str =
    "of each dependency, helping you understand the libraries your project uses ";
const README_ABOUT_P4: &str = "without having to browse through the original source code.\n\n";
const README_ABOUT_P5: &str =
    "Each file contains a summary of the corresponding dependency's source code, ";
const README_ABOUT_P6: &str = "including important types, functions, and structures.\n\n";
const README_ABOUT_P7: &str = "Generated by [DepBank](https://github.com/tyrchen/depbank).\n";

pub fn generate_command(project_path: &Path, output_dir: &Path, dry_run: bool) -> Result<()> {
    println!("Analyzing project...");

    // Find and analyze dependencies
    let available_deps = analyze_dependencies(project_path, dry_run)?;

    if dry_run {
        println!("Dry run enabled, skipping generation");
        return Ok(());
    }

    // Generate code banks
    println!("Generating code banks...");
    let registry_path = resolve_registry_path()?;
    let code_bank_files = generate_all_code_banks(&available_deps, &registry_path, output_dir)?;
    println!("Generated {} code bank files", code_bank_files.len());

    // Calculate tokens and generate README
    generate_code_bank_readme(
        output_dir,
        project_path,
        &available_deps,
        code_bank_files.len(),
    )?;

    Ok(())
}

fn analyze_dependencies(project_path: &Path, _dry_run: bool) -> Result<DependencyCollection> {
    // Find all Cargo.toml files
    let cargo_toml_files = find_cargo_toml_files(project_path)?;
    println!("Found {} Cargo.toml files", cargo_toml_files.len());

    if cargo_toml_files.is_empty() {
        return Err(anyhow::anyhow!("No Cargo.toml files found"));
    }

    // Extract dependencies from *all* found Cargo.toml files
    let mut dependency_info = DependencyCollection::new();
    let mut unique_deps_for_reporting = HashSet::new(); // Track unique dep names for reporting

    for cargo_toml_path in &cargo_toml_files {
        let file_deps = extract_dependency_info(cargo_toml_path)?;
        for dep in file_deps.iter() {
            // Add to the main collection
            dependency_info.add(dep.clone());
            // Track unique names for reporting count
            unique_deps_for_reporting.insert(dep.name.clone());
        }
    }

    println!(
        "Found {} unique dependencies",
        unique_deps_for_reporting.len()
    );

    // Find Cargo.lock - should still be at the workspace root
    let cargo_lock_path = find_cargo_lock(project_path)?;
    println!("Found Cargo.lock");

    // Resolve exact versions from Cargo.lock using the aggregated dependency info
    let resolved_versions = resolve_dependency_versions(cargo_lock_path, &dependency_info)?;
    println!("Resolved {} versions", resolved_versions.len());

    // Resolve registry path
    let registry_path = resolve_registry_path()?;

    // Check which dependencies are available locally
    let mut available_deps = DependencyCollection::new();
    // Keep track of names we've already added to available_deps to avoid duplicates if
    // the same dep name appears in multiple Cargo.toml files after version resolution.
    let mut added_names = HashSet::new();
    for dependency in resolved_versions.iter() {
        if !added_names.contains(&dependency.name)
            && is_dependency_available(&registry_path, dependency)
        {
            available_deps.add(dependency.clone());
            added_names.insert(dependency.name.clone());
        }
    }

    println!(
        "{}/{} unique dependencies available locally",
        available_deps.len(),
        unique_deps_for_reporting.len() // Report against unique names found initially
    );

    if available_deps.is_empty() {
        // Adjust the error message to be more accurate in a workspace context
        return Err(anyhow::anyhow!(
            "No resolved dependencies available locally in the Cargo registry"
        ));
    }

    // Return the available dependencies
    Ok(available_deps)
}

fn generate_code_bank_readme(
    output_dir: &Path,
    project_path: &Path,
    dependencies: &DependencyCollection,
    code_bank_files_count: usize,
) -> Result<()> {
    println!("Calculating tokens for generated code banks (may take a while)...");
    // Calculate tokens for generated code banks
    let file_stats = calculate_directory_tokens(output_dir, Some("md"))?;

    // Sort stats by token count
    let mut stats_vec: Vec<_> = file_stats.iter().collect();
    stats_vec.sort_by(|a, b| b.1.token_count.cmp(&a.1.token_count)); // Sort by token count (descending)

    // Generate README content
    let (readme_content, total_tokens) = create_readme_content(
        project_path,
        code_bank_files_count,
        &stats_vec,
        dependencies,
        &file_stats,
    );

    // Write README.md to the output directory
    let readme_path = output_dir.join("README.md");
    fs::write(&readme_path, readme_content).with_context(|| {
        format!(
            "Failed to write README.md to file: {}",
            readme_path.display()
        )
    })?;

    // Print concise summary to console
    println!("\nSummary:");
    println!("- Generated {} code bank files", code_bank_files_count);
    println!("- Total tokens: {}", total_tokens);
    println!("- Added README.md with summary and token information");
    println!("- Output directory: {}", output_dir.display());

    Ok(())
}

fn create_readme_content(
    project_path: &Path,
    code_bank_files_count: usize,
    stats_vec: &[(&String, &depbank::FileStats)],
    dependencies: &DependencyCollection,
    file_stats: &HashMap<String, depbank::FileStats>,
) -> (String, usize) {
    let mut total_tokens = 0;
    let mut total_size = 0;
    let mut readme_content = String::new();

    // Add header
    readme_content.push_str(README_HEADER);
    write!(
        readme_content,
        "Generated for project: {}\n\n",
        project_path.display()
    )
    .unwrap();

    // Add dependency section
    write!(
        readme_content,
        "## Dependencies ({} total)\n\n",
        code_bank_files_count
    )
    .unwrap();

    // Add dependency list with versions
    readme_content.push_str(README_TABLE_HEADER);
    readme_content.push_str(README_TABLE_SEPARATOR);

    for (name, stats) in stats_vec {
        let name_without_md = name.trim_end_matches(".md");
        let version = dependencies
            .get_version(name_without_md)
            .map(|v| v.as_str())
            .unwrap_or("unknown");

        writeln!(
            readme_content,
            "| {} | {} | {} | {} |",
            name_without_md, version, stats.token_count, stats.size_bytes
        )
        .unwrap();

        total_tokens += stats.token_count;
        total_size += stats.size_bytes;
    }

    // Add token summary
    readme_content.push_str(README_TOKEN_SUMMARY_HEADER);
    writeln!(readme_content, "- **Total tokens:** {}", total_tokens).unwrap();
    writeln!(readme_content, "- **Total size:** {} bytes", total_size).unwrap();
    writeln!(
        readme_content,
        "- **Number of files:** {}",
        file_stats.len()
    )
    .unwrap();

    // Add explanation of what code banks are
    readme_content.push_str(README_ABOUT_HEADER);
    readme_content.push_str(README_ABOUT_P1);
    readme_content.push_str(README_ABOUT_P2);
    readme_content.push_str(README_ABOUT_P3);
    readme_content.push_str(README_ABOUT_P4);
    readme_content.push_str(README_ABOUT_P5);
    readme_content.push_str(README_ABOUT_P6);
    readme_content.push_str(README_ABOUT_P7);

    (readme_content, total_tokens)
}

pub fn tokens_command(path: &Path, extension: Option<&str>) -> Result<()> {
    if path.is_file() {
        analyze_file_tokens(path)?;
    } else if path.is_dir() {
        analyze_directory_tokens(path, extension)?;
    } else {
        return Err(anyhow::anyhow!(
            "Path does not exist or is not accessible: {}",
            path.display()
        ));
    }

    Ok(())
}

fn analyze_file_tokens(path: &Path) -> Result<()> {
    // Calculate tokens for a single file
    let token_count = calculate_file_tokens(path)?;
    let file_size = std::fs::metadata(path)?.len();
    println!(
        "{}: {} tokens, {} bytes",
        path.display(),
        token_count,
        file_size
    );
    Ok(())
}

fn analyze_directory_tokens(dir_path: &Path, extension: Option<&str>) -> Result<()> {
    // Calculate tokens for all files in the directory
    let file_stats = calculate_directory_tokens(dir_path, extension)?;

    // Print token counts in a sorted manner
    let mut stats_vec: Vec<_> = file_stats.iter().collect();
    stats_vec.sort_by(|a, b| b.1.token_count.cmp(&a.1.token_count)); // Sort by token count (descending)

    let (total_tokens, total_size) = print_token_stats(dir_path, &stats_vec, file_stats.len());

    println!(
        "\nTotal: {} tokens, {} bytes across {} files",
        total_tokens,
        total_size,
        file_stats.len()
    );

    Ok(())
}

fn print_token_stats(
    dir_path: &Path,
    stats_vec: &[(&String, &depbank::FileStats)],
    _file_count: usize,
) -> (usize, usize) {
    let mut total_tokens = 0;
    let mut total_size = 0;

    println!("Token counts for files in {}:", dir_path.display());
    for (name, stats) in stats_vec {
        println!(
            "{}: {} tokens, {} bytes",
            name, stats.token_count, stats.size_bytes
        );

        total_tokens += stats.token_count;
        total_size += stats.size_bytes;
    }

    (total_tokens, total_size)
}

pub fn list_command(project_path: &Path, detailed: bool) -> Result<()> {
    // Find all Cargo.toml files
    let cargo_toml_files = find_cargo_toml_files(project_path)?;
    println!("Found {} Cargo.toml files", cargo_toml_files.len());

    if cargo_toml_files.is_empty() {
        return Err(anyhow::anyhow!("No Cargo.toml files found"));
    }

    // Collect all dependencies
    let dependencies = collect_dependencies(&cargo_toml_files)?;
    println!("\nFound {} unique dependencies:", dependencies.len());

    if detailed {
        display_detailed_dependency_info(project_path, &cargo_toml_files)?;
    } else {
        display_simple_dependency_list(&dependencies);
    }

    Ok(())
}

fn display_simple_dependency_list(dependencies: &HashSet<String>) {
    // Sort dependencies for consistent output
    let mut sorted_deps: Vec<_> = dependencies.iter().collect();
    sorted_deps.sort();

    // For simple view, just list dependencies
    for dep in sorted_deps {
        println!("- {}", dep);
    }
}

fn display_detailed_dependency_info(
    project_path: &Path,
    cargo_toml_files: &[PathBuf],
) -> Result<()> {
    // For detailed view, show dependency info from each Cargo.toml
    display_dependency_specs_by_file(cargo_toml_files)?;

    // Try to resolve versions from Cargo.lock if available
    display_cargo_lock_versions(project_path, cargo_toml_files)?;

    Ok(())
}

fn display_dependency_specs_by_file(cargo_toml_files: &[PathBuf]) -> Result<()> {
    for (index, cargo_toml) in cargo_toml_files.iter().enumerate() {
        println!("\nDependency specifications from {}:", cargo_toml.display());

        let dependency_info = extract_dependency_info(cargo_toml)?;

        // Sort dependencies for consistent output
        let mut sorted_info: Vec<_> = dependency_info.iter().collect();
        sorted_info.sort_by(|a, b| a.version.cmp(&b.version));

        for dep in sorted_info {
            println!("{}: {}", dep.name, dep.version);
        }

        // If not the last Cargo.toml, add a separator
        if index < cargo_toml_files.len() - 1 {
            println!("\n---");
        }
    }

    Ok(())
}

fn display_cargo_lock_versions(project_path: &Path, cargo_toml_files: &[PathBuf]) -> Result<()> {
    if let Ok(cargo_lock_path) = find_cargo_lock(project_path) {
        println!("\nFound Cargo.lock at: {}", cargo_lock_path.display());

        // Extract dependencies from the first Cargo.toml for resolution
        let first_cargo_toml = &cargo_toml_files[0];
        let dependency_info = extract_dependency_info(first_cargo_toml)?;

        // Resolve exact versions from Cargo.lock
        if let Ok(resolved_versions) =
            resolve_dependency_versions(cargo_lock_path, &dependency_info)
        {
            println!("\nResolved dependency versions from Cargo.lock:");

            // Sort dependencies for consistent output
            let mut sorted_resolved: Vec<_> = resolved_versions.iter().collect();
            sorted_resolved.sort_by(|a, b| a.version.cmp(&b.version));

            for dep in sorted_resolved {
                println!("{}: {}", dep.name, dep.version);
            }
        }
    }

    Ok(())
}