anything-to-skill 0.1.2

Turn books and documents into agent skills — one binary, every parser built in
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

//! The run report — `metadata.json` plus the terminal summary.
//!
//! This file is the handoff to the model. Everything the skill needs to decide
//! what to do next has to be in here, especially the parts that went wrong:
//! pages that could not be read are the difference between a skill with a hole
//! in it and a skill that knows it has a hole.

use crate::extract::Extraction;
use crate::{structure, tokens};
use serde::Serialize;
use std::path::Path;

#[derive(Debug, Serialize)]
pub struct FileReport {
    pub path: String,
    pub method: String,
    pub characters: usize,
    pub estimated_tokens: usize,
    pub page_count: Option<u32>,
    /// Pages whose text is unreliable. The skill renders these to images.
    pub pages_needing_ocr: Vec<u32>,
    pub pages_with_tables: Vec<u32>,
    pub pages_with_columns: Vec<u32>,
    /// Invisible code points removed as potential prompt injection.
    pub invisible_codepoints_removed: usize,
}

impl FileReport {
    pub fn new(path: &Path, extraction: &Extraction, text: &str, removed: usize) -> Self {
        FileReport {
            path: path.display().to_string(),
            method: extraction.method.clone(),
            characters: text.chars().count(),
            estimated_tokens: tokens::estimate(text),
            page_count: extraction.page_count,
            pages_needing_ocr: extraction.pages_needing_ocr.clone(),
            pages_with_tables: extraction.pages_with_tables.clone(),
            pages_with_columns: extraction.pages_with_columns.clone(),
            invisible_codepoints_removed: removed,
        }
    }
}

#[derive(Debug, Serialize)]
pub struct RunReport {
    pub output_text: String,
    pub characters: usize,
    pub estimated_tokens: usize,
    pub structure: structure::Structure,
    pub files: Vec<FileReport>,
    /// Files that produced nothing, with the reason. Empty on a clean run.
    pub failures: Vec<String>,
    /// Pages the model should read as images, grouped by source file. Present
    /// only when there is something to do, so the skill can branch on its
    /// existence rather than parsing an empty structure.
    #[serde(skip_serializing_if = "Vec::is_empty")]
    pub needs_visual_reading: Vec<VisualReadRequest>,
}

#[derive(Debug, Serialize)]
pub struct VisualReadRequest {
    pub path: String,
    pub pages: Vec<u32>,
}

impl RunReport {
    pub fn new(
        text_path: &Path,
        combined: &str,
        files: Vec<FileReport>,
        failures: Vec<String>,
    ) -> Self {
        let needs_visual_reading = files
            .iter()
            .filter(|f| !f.pages_needing_ocr.is_empty())
            .map(|f| VisualReadRequest {
                path: f.path.clone(),
                pages: f.pages_needing_ocr.clone(),
            })
            .collect();

        RunReport {
            output_text: text_path.display().to_string(),
            characters: combined.chars().count(),
            estimated_tokens: tokens::estimate(combined),
            structure: structure::detect(combined),
            files,
            failures,
            needs_visual_reading,
        }
    }

    pub fn print_summary(&self, text_path: &Path, meta_path: &Path) {
        eprintln!();
        eprintln!("extracted {} file(s)", self.files.len());
        eprintln!(
            "  {} characters, ~{} tokens",
            self.characters, self.estimated_tokens
        );
        eprintln!(
            "  {} chapter(s) detected{}",
            self.structure.chapters_detected,
            if self.structure.has_toc {
                ", table of contents present"
            } else {
                ""
            }
        );
        for file in &self.files {
            eprintln!("  {} — {}", file.path, file.method);
        }
        if !self.needs_visual_reading.is_empty() {
            let total: usize = self
                .needs_visual_reading
                .iter()
                .map(|r| r.pages.len())
                .sum();
            eprintln!();
            eprintln!(
                "  {total} page(s) could not be read as text and are missing from the output."
            );
            eprintln!("  Render and read them with:");
            for request in &self.needs_visual_reading {
                let pages: Vec<String> = request.pages.iter().map(u32::to_string).collect();
                eprintln!(
                    "    anything-to-skill render '{}' --pages {} --out <dir>",
                    request.path,
                    pages.join(",")
                );
            }
        }
        if !self.failures.is_empty() {
            eprintln!();
            eprintln!("  {} file(s) failed:", self.failures.len());
            for failure in &self.failures {
                eprintln!("    {failure}");
            }
        }
        eprintln!();
        eprintln!("  text     {}", text_path.display());
        eprintln!("  metadata {}", meta_path.display());
    }
}