1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
use serde::Serialize;
/// Details about how extraction selected, cleaned, and accepted article roots.
#[derive(Clone, Debug, Default, Serialize, PartialEq)]
pub struct ExtractionDiagnostics {
/// Result of applying a caller-provided `content_selector`.
pub content_selector: Option<ContentSelectorDiagnostic>,
/// Result of applying a bundled or caller-provided site profile.
pub site_rule: Option<SiteRuleDiagnostic>,
/// Generic extraction attempts tried after profile and selector handling.
pub attempts: Vec<AttemptDiagnostic>,
/// Index into `attempts` for the selected generic extraction attempt.
pub selected_attempt: Option<usize>,
/// Final extraction outcome.
pub outcome: ExtractionOutcome,
}
/// Final status for an extraction report.
#[derive(Clone, Debug, Default, Serialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum ExtractionOutcome {
/// Extraction met the configured acceptance thresholds.
Accepted,
/// Extraction returned the best available attempt below normal thresholds.
BestAttempt,
/// Extraction found no usable article content.
#[default]
NoContent,
}
/// Article output plus diagnostics for the decisions made during extraction.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct ExtractionReport {
/// Extracted article, if any content was accepted.
pub article: Option<crate::Article>,
/// Root selection, cleanup, and fallback diagnostics.
pub diagnostics: ExtractionDiagnostics,
}
/// Diagnostics for a caller-provided content selector.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct ContentSelectorDiagnostic {
/// CSS selector supplied by the caller.
pub selector: String,
/// Whether the selector matched at least one node.
pub matched: bool,
/// Node selected for extraction when a match was usable.
pub selected: Option<NodeDiagnostic>,
}
/// Diagnostics for a matched site-specific extraction rule.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct SiteRuleDiagnostic {
/// Profile or extractor name.
pub name: String,
/// Where the rule came from.
pub source: SiteRuleSource,
/// Host and path match that selected this rule.
pub matched_by: SiteRuleMatch,
/// CSS-like selectors for roots selected by the rule.
pub roots: Vec<String>,
/// Number of elements removed by the rule before cleanup.
pub removals: usize,
/// Text length after rule extraction.
pub text_len: usize,
/// Whether the rule output met acceptance thresholds.
pub accepted: bool,
/// Reason generic extraction was used after a weak rule result.
pub fallback_reason: Option<String>,
}
/// Source of a site-specific extraction rule.
#[derive(Clone, Debug, Serialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum SiteRuleSource {
/// A TOML profile selected the content.
DeclarativeProfile,
/// A Rust extractor handled the site.
CodeExtractor,
}
/// Host and path rule that matched a page URL.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct SiteRuleMatch {
/// Hostname matched by the rule.
pub host: String,
/// Optional path prefix required by the rule.
pub path_prefix: Option<String>,
/// Whether the matching rule is bundled with Lectito.
pub bundled: bool,
}
/// Diagnostics for one generic extraction attempt.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct AttemptDiagnostic {
/// Attempt index in retry order.
pub index: usize,
/// Cleanup and scoring flags used for this attempt.
pub flags: FlagDiagnostic,
/// Number of scored candidate roots.
pub candidate_count: usize,
/// Highest-scoring generic candidate roots.
pub candidates: Vec<CandidateDiagnostic>,
/// Candidate roots found through article-like entry points.
pub entry_points: Vec<CandidateDiagnostic>,
/// Root selected for cleanup and serialization.
pub selected_root: Option<NodeDiagnostic>,
/// Cleanup result for the selected root.
pub cleanup: Option<CleanupDiagnostic>,
/// Content recovered from hidden or alternate markup before scoring.
pub recovery: RecoveryDiagnostic,
/// Extracted text length after cleanup.
pub text_len: usize,
/// Whether this attempt met acceptance thresholds.
pub accepted: bool,
}
/// Generic extraction flags used for an attempt.
#[derive(Clone, Copy, Debug, Serialize, PartialEq)]
pub struct FlagDiagnostic {
/// Whether unlikely page chrome was stripped before scoring.
pub strip_unlikely: bool,
/// Whether positive and negative class names affected scoring.
pub weight_classes: bool,
/// Whether low-value child nodes were removed conditionally.
pub clean_conditionally: bool,
}
/// Diagnostics for one candidate article root.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct CandidateDiagnostic {
/// Candidate node summary.
pub node: NodeDiagnostic,
/// Readability score assigned to the candidate.
pub score: f64,
/// How this candidate entered the selection set.
pub selected_by: CandidateSelection,
}
/// Reason a node was considered as an article root.
#[derive(Clone, Debug, Serialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum CandidateSelection {
/// The node was found by generic readability scoring.
CandidateScoring,
/// The node matched an article-like entry point before scoring.
EntryPointPreselection,
}
/// Stable summary of a DOM node used in diagnostics.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct NodeDiagnostic {
/// CSS-like selector for the node.
pub selector: String,
/// HTML tag name.
pub tag: String,
/// Element id, if present.
pub id: Option<String>,
/// Element classes.
pub classes: Vec<String>,
/// Visible text length under the node.
pub text_len: usize,
/// Link text ratio under the node.
pub link_density: f64,
}
/// Cleanup measurements for a selected article root.
#[derive(Clone, Debug, Serialize, PartialEq)]
pub struct CleanupDiagnostic {
/// Root selectors cleaned for output.
pub roots: Vec<String>,
/// Text length before cleanup.
pub text_len_before: usize,
/// Text length after cleanup.
pub text_len_after: usize,
/// Element count before cleanup.
pub element_count_before: usize,
/// Element count after cleanup.
pub element_count_after: usize,
/// Number of elements removed during cleanup.
pub removed_elements: usize,
}
/// Content recovery performed before scoring.
#[derive(Clone, Debug, Default, Serialize, PartialEq)]
pub struct RecoveryDiagnostic {
/// Number of declarative shadow roots flattened into normal markup.
pub shadow_roots_flattened: usize,
/// Number of mobile-only style rules applied to recover hidden content.
pub mobile_rules_applied: usize,
}