html_helpers/slimmer/
slim.rs

1use super::SlimOptions;
2use crate::error::{Error, Result};
3use ego_tree::NodeRef;
4use scraper::{ElementRef, Html, node::Node};
5
6use super::support::{
7	BLOCK_LEVEL_TAGS, REMOVABLE_EMPTY_TAGS, TAGS_TO_REMOVE, VOID_ELEMENTS, filter_and_write_attributes,
8	is_string_effectively_empty, remove_empty_lines, should_keep_meta,
9};
10
11/// Decodes HTML entities (e.g., `&lt;` becomes `<`).
12/// Re-exporting from the original slimmer or using html-escape directly.
13pub fn decode_html_entities(content: &str) -> String {
14	html_escape::decode_html_entities(content).to_string()
15}
16
17/// Strips non-content elements from the provided HTML content using the `scraper` crate,
18/// preserving essential head tags, and returns the cleaned HTML as a string.
19///
20/// This function aims to replicate the behavior of `slimmer::slim` using `scraper`.
21/// It removes:
22/// - Non-visible tags like `<script>`, `<link>`, `<style>`, `<svg>`, `<base>`.
23/// - HTML comments.
24/// - Empty or whitespace-only text nodes.
25/// - Specific tags (like `<div>`, `<span>`, `<p>`, etc.) if they become effectively empty *after* processing children.
26/// - Attributes except for specific allowlists (`class`, `aria-label`, `href` outside head; `property`, `content` for relevant meta tags in head).
27///
28/// It preserves:
29/// - `<title>` tag within `<head>`.
30/// - `<meta>` tags within `<head>` if their `property` attribute matches keywords in `META_PROPERTY_KEYWORDS`.
31/// - Essential body content.
32///
33/// # Arguments
34///
35/// * `html_content` - A string slice containing the HTML content to be processed.
36///
37/// # Returns
38///
39/// A `Result<String>` which is:
40/// - `Ok(String)` containing the cleaned HTML content.
41/// - `Err` if any errors occur during processing.
42pub fn slim(html_content: &str, options: impl Into<SlimOptions>) -> Result<String> {
43	let options = options.into();
44	let html = Html::parse_document(html_content);
45	let mut output = String::new();
46
47	process_node_stack_based(html.tree.root(), false, &options, 0, &mut output)?;
48
49	// Final cleanup of empty lines
50	let content = remove_empty_lines(output)?;
51
52	Ok(content)
53}
54
55/// Non‑recursive stack‑based version of the slim processing.
56fn process_node_stack_based(
57	root_node: NodeRef<Node>,
58	is_in_head_context: bool,
59	options: &SlimOptions,
60	depth: usize,
61	output: &mut String,
62) -> Result<()> {
63	let indent_spaces = options.indent.unwrap_or(0) as usize;
64	let use_tabs = options.indent_with_tabs;
65
66	#[derive(Clone)]
67	enum FrameState {
68		Enter,
69		Exit,
70	}
71
72	struct Frame<'a> {
73		node: NodeRef<'a, Node>,
74		is_in_head_context: bool,
75		depth: usize,
76		state: FrameState,
77		children_output: String,
78		/// Where this frame's output should be appended.
79		/// `Some(idx)` means the frame at the given stack index is the parent
80		/// that will collect our output; `None` means append to global output.
81		output_target_index: Option<usize>,
82	}
83
84	let mut stack: Vec<Frame> = Vec::new();
85	stack.push(Frame {
86		node: root_node,
87		is_in_head_context,
88		depth,
89		state: FrameState::Enter,
90		children_output: String::new(),
91		output_target_index: None,
92	});
93
94	while let Some(frame) = stack.pop() {
95		match frame.state {
96			FrameState::Enter => {
97				match frame.node.value() {
98					Node::Document => {
99						// Push children in reverse order (no Exit needed)
100						let children: Vec<_> = frame.node.children().collect();
101						for child in children.into_iter().rev() {
102							stack.push(Frame {
103								node: child,
104								is_in_head_context: false,
105								depth: frame.depth,
106								state: FrameState::Enter,
107								children_output: String::new(),
108								output_target_index: frame.output_target_index,
109							});
110						}
111					}
112					Node::Doctype(doctype) => {
113						// Serialize Doctype
114						let mut s = String::new();
115						s.push_str("<!DOCTYPE ");
116						s.push_str(&doctype.name);
117						let has_public = !doctype.public_id.is_empty();
118						let has_system = !doctype.system_id.is_empty();
119
120						if has_public {
121							s.push_str(" PUBLIC \"");
122							s.push_str(&doctype.public_id);
123							s.push('"');
124						}
125
126						if has_system {
127							if !has_public {
128								s.push_str(" SYSTEM");
129							}
130							s.push(' ');
131							s.push('"');
132							s.push_str(&doctype.system_id);
133							s.push('"');
134						}
135						s.push('>');
136
137						if indent_spaces > 0 {
138							s.push('\n');
139						}
140
141						// Append to parent frame or global output
142						match frame.output_target_index {
143							Some(idx) => {
144								stack
145									.get_mut(idx)
146									.expect("target frame should exist")
147									.children_output
148									.push_str(&s);
149							}
150							None => {
151								output.push_str(&s);
152							}
153						}
154					}
155					Node::Comment(_) => { /* Skip comments */ }
156					Node::Text(text) => {
157						let text_content = text.trim();
158						if !text_content.is_empty() {
159							let s = text.to_string();
160							match frame.output_target_index {
161								Some(idx) => {
162									stack
163										.get_mut(idx)
164										.expect("target frame should exist")
165										.children_output
166										.push_str(&s);
167								}
168								None => {
169									output.push_str(&s);
170								}
171							}
172						}
173					}
174					Node::Element(element) => {
175						let tag_name = element.name();
176
177						// Handle <html> as transparent container: push children directly, no wrapper.
178						if tag_name == "html" {
179							let child_context_is_in_head = frame.is_in_head_context;
180							let mut children: Vec<_> = frame.node.children().collect();
181							children.reverse();
182							for child in children {
183								stack.push(Frame {
184									node: child,
185									is_in_head_context: child_context_is_in_head,
186									depth: frame.depth,
187									state: FrameState::Enter,
188									children_output: String::new(),
189									output_target_index: frame.output_target_index,
190								});
191							}
192							continue;
193						}
194
195						let el_ref = ElementRef::wrap(frame.node)
196							.ok_or_else(|| Error::custom("Failed to wrap node as ElementRef"))?;
197
198						let current_node_is_head = tag_name == "head";
199						let child_context_is_in_head = frame.is_in_head_context || current_node_is_head;
200
201						// Fast-skip rules
202						// Fast-skip rules
203						let should_skip = match tag_name {
204							_ if !child_context_is_in_head && TAGS_TO_REMOVE.contains(&tag_name) => true,
205							"script" | "style" | "link" | "base" | "svg" => true,
206							_ if frame.is_in_head_context => {
207								!(tag_name == "title" || (tag_name == "meta" && should_keep_meta(el_ref)))
208							}
209							_ => false,
210						};
211
212						if should_skip {
213							continue;
214						}
215
216						// Push Exit frame for this element
217						let exit_idx = stack.len();
218						stack.push(Frame {
219							node: frame.node,
220							is_in_head_context: frame.is_in_head_context,
221							depth: frame.depth,
222							state: FrameState::Exit,
223							children_output: String::new(),
224							output_target_index: frame.output_target_index,
225						});
226
227						// Compute child depth and push children in reverse order
228						let is_formatting = indent_spaces > 0 || use_tabs;
229						let is_block = if is_formatting {
230							BLOCK_LEVEL_TAGS.contains(&tag_name) || tag_name == "title"
231						} else {
232							false
233						};
234						let child_depth = if is_block { frame.depth + 1 } else { frame.depth };
235
236						let mut children: Vec<_> = frame.node.children().collect();
237						children.reverse();
238						for child in children {
239							stack.push(Frame {
240								node: child,
241								is_in_head_context: child_context_is_in_head,
242								depth: child_depth,
243								state: FrameState::Enter,
244								children_output: String::new(),
245								output_target_index: Some(exit_idx),
246							});
247						}
248					}
249					Node::Fragment => {
250						let children: Vec<_> = frame.node.children().collect();
251						for child in children.into_iter().rev() {
252							stack.push(Frame {
253								node: child,
254								is_in_head_context: false,
255								depth: frame.depth,
256								state: FrameState::Enter,
257								children_output: String::new(),
258								output_target_index: frame.output_target_index,
259							});
260						}
261					}
262					Node::ProcessingInstruction(_) => { /* Skip PIs */ }
263				}
264			}
265			FrameState::Exit => {
266				let el_ref =
267					ElementRef::wrap(frame.node).ok_or_else(|| Error::custom("Failed to wrap node as ElementRef"))?;
268				let tag_name = el_ref.value().name();
269
270				let is_formatting = indent_spaces > 0 || use_tabs;
271				let is_block = if is_formatting {
272					BLOCK_LEVEL_TAGS.contains(&tag_name) || tag_name == "title"
273				} else {
274					false
275				};
276				let is_void = is_formatting && VOID_ELEMENTS.contains(&tag_name);
277
278				let is_empty_after_processing = is_string_effectively_empty(&frame.children_output);
279				let is_in_head_for_removal = frame.is_in_head_context || tag_name == "head";
280				let is_removable_tag_when_empty = !is_in_head_for_removal && REMOVABLE_EMPTY_TAGS.contains(&tag_name);
281				let is_empty_head_tag = tag_name == "head" && is_empty_after_processing;
282				let should_remove = (is_removable_tag_when_empty && is_empty_after_processing) || is_empty_head_tag;
283
284				if should_remove {
285					continue;
286				}
287
288				let mut out = String::new();
289
290				// Indent before opening tag (block‑level)
291				if is_block {
292					out.push('\n');
293					let indent_str = if use_tabs {
294						"\t".repeat(frame.depth)
295					} else {
296						" ".repeat(frame.depth * indent_spaces)
297					};
298					out.push_str(&indent_str);
299				}
300
301				// Start tag with filtered attributes
302				out.push('<');
303				out.push_str(tag_name);
304				// Attribute filter uses the head‑context of the element itself
305				let is_in_head_for_attrs = frame.is_in_head_context || tag_name == "head";
306				filter_and_write_attributes(el_ref, is_in_head_for_attrs, &mut out)?;
307				out.push('>');
308
309				// Append children output
310				out.push_str(&frame.children_output);
311
312				// Indent before closing tag if needed
313				if is_block && !is_void && frame.children_output.contains('\n') {
314					out.push('\n');
315					let indent_str = if use_tabs {
316						"\t".repeat(frame.depth)
317					} else {
318						" ".repeat(frame.depth * indent_spaces)
319					};
320					out.push_str(&indent_str);
321				}
322
323				// Closing tag unless void
324				if !is_void {
325					out.push_str("</");
326					out.push_str(tag_name);
327					out.push('>');
328				}
329
330				// Append to parent frame or global output
331				match frame.output_target_index {
332					Some(idx) => {
333						stack
334							.get_mut(idx)
335							.expect("target frame should exist")
336							.children_output
337							.push_str(&out);
338					}
339					None => {
340						output.push_str(&out);
341					}
342				}
343			}
344		}
345	}
346
347	Ok(())
348}
349
350// region:    --- Tests
351
352#[cfg(test)]
353mod tests {
354	use super::*;
355	// Result type alias for tests
356	type TestResult<T> = core::result::Result<T, Box<dyn std::error::Error>>;
357
358	// Copied and adapted tests from slimmer.rs
359	// Renamed slim -> slim2 and test_slimmer_... -> test_slimmer2_...
360
361	#[test]
362	fn test_slimmer2_slim_basic() -> TestResult<()> {
363		// -- Setup & Fixtures
364		let fx_html = r#"
365<!DOCTYPE html>
366<html lang="en">
367<head>
368    <meta charset="UTF-8">
369    <meta name="viewport" content="width=device-width, initial-scale=1.0">
370	<meta property="og:title" content="Test Title">
371	<meta property="og:url" content="http://example.com">
372	<meta property="og:image" content="http://example.com/img.png">
373	<meta property="og:description" content="Test Description">
374	<meta name="keywords" content="test, html"> <!-- Should be removed -->
375    <title>Simple HTML Page</title>
376	<style> body{ color: red } </style>
377	<link rel="stylesheet" href="style.css">
378	<script> console.log("hi"); </script>
379	<base href="/"> <!-- Should be removed -->
380</head>
381<body class="main-body" aria-label="Page body">
382	<svg><path d="M0 0 L 10 10"></path></svg> <!-- Should be removed -->
383	<div>
384		<span></span> <!-- Should be removed (effectively empty after processing) -->
385		<p> <!-- Effectively empty after processing --> </p>
386		<b>  </b> <!-- Effectively empty after processing -->
387		<i><!-- comment --></i> <!-- Effectively empty after processing -->
388	</div> <!-- Should be removed (effectively empty after children removed) -->
389	<section>Content Inside</section> <!-- Should be kept -->
390	<article>  </article> <!-- Should be removed (empty after processing) -->
391    <h1 funky-attribute="removeme">Hello, World!</h1> <!-- funky-attribute removed -->
392    <p>This is a simple HTML page.</p>
393	<a href="https://example.org" class="link-style" extra="gone">Link</a> <!-- href and class kept -->
394	<!-- Some Comment -->
395</body>
396</html>
397		"#;
398
399		// Expected output should now match slimmer.rs more closely regarding empty element removal.
400		// let expected_head_content = r#"<head><meta content="Test Title" property="og:title"><meta content="http://example.com" property="og:url"><meta content="http://example.com/img.png" property="og:image"><meta content="Test Description" property="og:description"><title>Simple HTML Page</title></head>"#;
401		let expected_body_content = r#"<body aria-label="Page body" class="main-body"><section>Content Inside</section><h1>Hello, World!</h1><p>This is a simple HTML page.</p><a class="link-style" href="https://example.org">Link</a></body>"#;
402		// Note attribute order might differ slightly between scraper/html5ever & string building, but content should match.
403
404		// -- Exec
405		let html = slim(fx_html, SlimOptions::default())?;
406		// println!(
407		// 	"\n---\nSlimmed HTML (Scraper - Basic + Post-Empty Removal):\n{}\n---\n",
408		// 	html
409		// );
410
411		// -- Check Head Content (More precise check possible now)
412		// Need flexible attribute order check for head
413		assert!(html.contains("<head>"));
414		assert!(html.contains("</head>"));
415		assert!(html.contains(r#"<meta content="Test Title" property="og:title">"#));
416		assert!(html.contains(r#"<meta content="http://example.com" property="og:url">"#));
417		assert!(html.contains(r#"<meta content="http://example.com/img.png" property="og:image">"#));
418		assert!(html.contains(r#"<meta content="Test Description" property="og:description">"#));
419		assert!(html.contains(r#"<title>Simple HTML Page</title>"#));
420
421		assert!(
422			!html.contains("<meta charset") && !html.contains("<meta name"),
423			"Should remove disallowed meta tags"
424		);
425		assert!(
426			!html.contains("<style") && !html.contains("<link") && !html.contains("<script") && !html.contains("<base"),
427			"Should remove style, link, script, base"
428		);
429
430		// -- Check Body Content (More precise check)
431		// Allow for attribute order variations in body tag
432		assert!(
433			html.contains("<body")
434				&& html.contains(r#"class="main-body""#)
435				&& html.contains(r#"aria-label="Page body""#)
436				&& html.contains(">")
437		);
438		assert!(html.contains(r#"</body>"#));
439		assert!(html.contains(expected_body_content)); // Check the exact sequence for the rest
440
441		// Check removals (should now match slimmer.rs)
442		assert!(!html.contains("<svg>"), "Should remove svg");
443		assert!(!html.contains("<span>"), "Should remove empty span");
444		assert!(!html.contains("<p> </p>"), "Should remove empty p tag");
445		assert!(!html.contains("<b>"), "Should remove empty b");
446		assert!(!html.contains("<i>"), "Should remove empty i");
447		assert!(!html.contains("<div>"), "Should remove outer empty div");
448		assert!(!html.contains("<article>"), "Should remove empty article");
449		assert!(!html.contains("funky-attribute"), "Should remove funky-attribute");
450		assert!(!html.contains("extra=\"gone\""), "Should remove extra anchor attribute");
451		assert!(!html.contains("<!--"), "Should remove comments");
452
453		Ok(())
454	}
455
456	#[test]
457	fn test_slimmer2_slim_empty_head_removed() -> TestResult<()> {
458		// -- Setup & Fixtures
459		let fx_html = r#"
460		<!DOCTYPE html>
461		<html>
462		<head>
463			<meta charset="utf-8">
464			<link rel="icon" href="favicon.ico">
465		</head>
466		<body>
467			<p>Content</p>
468		</body>
469		</html>
470		"#;
471
472		// -- Exec
473		let html = slim(fx_html, SlimOptions::default())?;
474		// println!("\n---\nSlimmed HTML (Scraper - Empty Head Removed):\n{}\n---\n", html);
475
476		// -- Check
477		// The <head> tag itself should now be removed as it becomes empty after processing children.
478		assert!(
479			!html.contains("<head>"),
480			"Empty <head> tag should be removed after processing. Got: {}",
481			html
482		);
483		assert!(html.contains("<body><p>Content</p></body>"), "Body should remain");
484
485		Ok(())
486	}
487
488	#[test]
489	fn test_slimmer2_slim_keeps_head_if_title_present() -> TestResult<()> {
490		// -- Setup & Fixtures
491		let fx_html = r#"
492		<!DOCTYPE html>
493		<html>
494		<head>
495			<title>Only Title</title>
496			<script></script>
497		</head>
498		<body>
499			<p>Content</p>
500		</body>
501		</html>
502		"#;
503
504		// -- Exec
505		let html = slim(fx_html, SlimOptions::default())?;
506		// println!("\n---\nSlimmed HTML (Scraper - Head with Title Kept):\n{}\n---\n", html);
507
508		// -- Check
509		// Head should remain as title is kept.
510		assert!(
511			html.contains("<head><title>Only Title</title></head>"),
512			"<head> with only title should remain"
513		);
514		assert!(!html.contains("<script>"), "Script should be removed");
515		assert!(html.contains("<body><p>Content</p></body>"), "Body should remain");
516
517		Ok(())
518	}
519
520	#[test]
521	fn test_slimmer2_slim_nested_empty_removal() -> TestResult<()> {
522		// -- Setup & Fixtures
523		let fx_html = r#"
524		<!DOCTYPE html>
525		<html>
526		<body>
527			<div> <!-- Will become empty after children removed -->
528				<p>  </p> <!-- empty p -->
529				<div> <!-- Inner div, will become empty -->
530					<span><!-- comment --></span> <!-- empty span -->
531				</div>
532			</div>
533			<section>
534				<h1>Title</h1> <!-- Keep H1 -->
535				<div> </div> <!-- Remove empty div -->
536			</section>
537		</body>
538		</html>
539		"#;
540		// Expected: Outer div removed, inner div removed, p removed, span removed. Section and H1 remain.
541		// This behaviour should now match html5ever version.
542		let expected_body = r#"<body><section><h1>Title</h1></section></body>"#;
543
544		// -- Exec
545		let html = slim(fx_html, SlimOptions::default())?;
546		// println!("\n---\nSlimmed HTML (Scraper - Nested Empty Removed):\n{}\n---\n", html);
547
548		// -- Check
549		assert!(
550			html.contains(expected_body),
551			"Should remove nested empty elements correctly after processing. Expected: '{}', Got: '{}'",
552			expected_body,
553			html
554		);
555		assert!(!html.contains("<p>"), "Empty <p> should be removed");
556		assert!(!html.contains("<span>"), "Empty <span> should be removed");
557		assert!(
558			!html.contains("<div>"),
559			"All empty <div> tags should be removed (inner and outer)"
560		);
561		assert!(html.contains("<section>"), "Section should remain");
562		assert!(html.contains("<h1>"), "H1 should remain");
563
564		Ok(())
565	}
566
567	#[test]
568	fn test_slimmer2_slim_keep_empty_but_not_removable() -> TestResult<()> {
569		// -- Setup & Fixtures
570		let fx_html = r#"
571		<!DOCTYPE html>
572		<html>
573		<body>
574			<main></main> <!-- Should keep 'main' even if empty -->
575			<table><tr><td></td></tr></table> <!-- Should keep table structure even if cells empty -->
576		</body>
577		</html>
578		"#;
579		let expected_body_fragment1 = "<main></main>";
580
581		// -- Exec
582		let html = slim(fx_html, SlimOptions::default())?;
583
584		// -- Check
585		assert!(html.contains(expected_body_fragment1), "Should keep empty <main>");
586		// Be flexible with tbody insertion
587		assert!(
588			html.contains("<table>") && html.contains("<tr>") && html.contains("<td>") && html.contains("</table>"),
589			"Should keep empty table structure. Got: {}",
590			html
591		);
592
593		Ok(())
594	}
595}
596
597// endregion: --- Tests
html_helpers/slimmer/slim.rs

html_helpers/slimmer/
slim.rs
