reget/
lib.rs

1//! # Recipe Extraction from HTML documents
2//!
3//! `reget` provides a [single function](parse_recipe) to extract a [recipe](Recipe) from HTML documents
4//! using structured data (JSON-LD) embedded within.
5//!
6//! With the optional `markdown` feature, recipes can be [converted to a markdown string](MarkdownBuilder).
7//!
8//! This library assumes the document follows the [schema.org recipe specification](https://schema.org/Recipe).
9//!
10//! ## Example
11//!
12//! ```
13//! use reget::parse_recipe;
14//!
15//! let html = r#"
16//! <!DOCTYPE html>
17//! <html lang="en">
18//! <script type="application/ld+json">
19//! {
20//!   "@type": "Recipe",
21//!   "name": "Delicious Cookies",
22//!   "author": "Lorem Ipsum",
23//!   "recipeIngredient": ["2 cups flour", "1 cup sugar"],
24//!   "recipeInstructions": "Mix ingredients and bake."
25//! }
26//! </script>
27//! </html>
28//! "#;
29//!
30//! let recipe = parse_recipe(html).unwrap();
31//!
32//! // And with the optional markdown feature    
33//! let md = recipe
34//!     .to_markdown()
35//!     .with_url("https://example.org/recipe")
36//!     .convert();
37//! ```
38
39mod constants;
40#[cfg(feature = "markdown")]
41mod markdown;
42mod model;
43
44use constants::LdFields;
45#[cfg(feature = "markdown")]
46pub use markdown::MarkdownBuilder;
47pub use model::{HowToSection, HowToStep, Ingredient, Recipe};
48
49use scraper::{Html, Selector};
50use serde_json::{Map, Value};
51
52const JSON_LD_SELECTOR: &str = r#"script[type="application/ld+json"]"#;
53const RECIPE_TYPE: &str = "Recipe";
54const HOW_TO_SECTION_TYPE: &str = "HowToSection";
55
56/// Parses the [recipe](Recipe) from the given HTML document. Will return None if no
57/// linked data is found in the document.
58///
59/// This function will only extract the first recipe it finds and only if it follows
60/// [schema.org recipe specification](https://schema.org/Recipe).
61///
62/// For an example see [here](crate).
63pub fn parse_recipe(html: &str) -> Option<Recipe> {
64    let json = extract_recipe_json(html)?;
65    Some(extract_recipe(&json))
66}
67
68fn extract_recipe(json: &Map<String, Value>) -> Recipe {
69    Recipe {
70        name: json
71            .get(LdFields::NAME)
72            .and_then(Value::as_str)
73            .map(String::from),
74        author: json.get(LdFields::AUTHOR).and_then(extract_author),
75        description: json
76            .get(LdFields::DESCRIPTION)
77            .and_then(Value::as_str)
78            .map(String::from),
79        ingredients: json
80            .get(LdFields::RECIPE_INGREDIENT)
81            .map(extract_ingredients)
82            .unwrap_or_default(),
83        how_to_sections: json
84            .get(LdFields::RECIPE_INSTRUCTIONS)
85            .map(extract_instructions)
86            .unwrap_or_default(),
87    }
88}
89
90/// Looks for `type="application/ld+json"` in the provided html with `"@type": Recipe`.
91fn extract_recipe_json(html: &str) -> Option<Map<String, Value>> {
92    let sel = Selector::parse(JSON_LD_SELECTOR).unwrap();
93    let document = Html::parse_document(html);
94
95    for e in document.select(&sel) {
96        let s = e.text().collect::<String>();
97
98        let value = match serde_json::from_str::<Value>(&s) {
99            Ok(val) => val,
100            Err(_) => continue, // parsing json failed
101        };
102
103        match find_recipe_in_value(value) {
104            Some(val) => return Some(val),
105            None => continue, // this is not the recipe
106        };
107    }
108    None
109}
110
111/// Tries to recursively find a recipe by looking for the tag `"@type": Recipe`.
112fn find_recipe_in_value(value: Value) -> Option<Map<String, Value>> {
113    match value {
114        Value::Object(obj) => {
115            if is_recipe_type(&obj) {
116                return Some(obj);
117            }
118            for (_, v) in obj {
119                if let Some(recipe) = find_recipe_in_value(v) {
120                    return Some(recipe);
121                }
122            }
123        }
124        Value::Array(arr) => {
125            for item in arr {
126                if let Some(recipe) = find_recipe_in_value(item) {
127                    return Some(recipe);
128                }
129            }
130        }
131        _ => {}
132    }
133    None
134}
135
136/// Verifies that the obj contains the tag `"@type": Recipe`.
137fn is_recipe_type(obj: &serde_json::Map<String, Value>) -> bool {
138    match obj.get(LdFields::TYPE) {
139        Some(Value::String(s)) => s == RECIPE_TYPE,
140        Some(Value::Array(arr)) => arr
141            .iter()
142            .any(|t| matches!(t, Value::String(type_str) if type_str == RECIPE_TYPE)),
143        _ => false,
144    }
145}
146
147/// Extracts the author
148///
149/// It deals with:
150///     - "author": "first last",
151///     - "author": { "name": "first last" },
152///     - "author": [ "first last", "first last" ]
153///     - "author": [ { "name": "first last" }, { "name": "first last" } ]
154///
155/// For arrays of authors it returns them as a comma separated string
156fn extract_author(value: &serde_json::Value) -> Option<String> {
157    match value {
158        // If the field is just a string, return the string
159        Value::String(str) => Some(str.clone()),
160        // If the field has a name field, return its value
161        Value::Object(obj) => match obj.get(LdFields::NAME) {
162            Some(Value::String(s)) => Some(s.clone()),
163            _ => None,
164        },
165        // If it is an array, return them as a comma seperated list
166        Value::Array(arr) => {
167            let strings = arr
168                .iter()
169                .filter_map(extract_author)
170                .collect::<Vec<String>>();
171            if strings.is_empty() {
172                None
173            } else {
174                Some(strings.join(", "))
175            }
176        }
177        _ => None,
178    }
179}
180
181/// Extracts the ingredients
182///
183/// It deals with:
184///     - "recipeIngredient": "ingredient",
185///     - "recipeIngredient": [ "ingredient1", "ingredient2" ]
186fn extract_ingredients(value: &serde_json::Value) -> Vec<Ingredient> {
187    match value {
188        Value::Array(arr) => arr
189            .iter()
190            .filter_map(|ingredient| match ingredient {
191                Value::String(s) => Some(s.clone()),
192                _ => None,
193            })
194            .collect(),
195        Value::String(s) => vec![s.to_string()],
196        _ => vec![],
197    }
198}
199
200/// Extracts the instructions
201///
202/// It deals with:
203///     - "recipeInstructions": "step text"
204///     - "recipeInstructions": [ "step1", "step2" ]
205///     - "recipeInstructions": [ { "text": "step1" }, { "text": "step2" } ]
206///     - "recipeInstructions": [ { "@type": "HowToSection", "name": "...", "itemListElement": [...] }, ... ]
207///
208/// For HowToSection objects, each section contains an array of steps.
209/// For arrays of steps or plain text, returns a single section with all steps concatenated.
210fn extract_instructions(value: &serde_json::Value) -> Vec<HowToSection> {
211    match value {
212        // Array of sections or steps
213        Value::Array(arr) => {
214            // If any item is a HowToSection, treat as sections
215            if arr.iter().any(is_how_to_section_obj) {
216                arr.iter().filter_map(extract_section).collect()
217            } else {
218                vec![HowToSection {
219                    name: None,
220                    steps: arr.iter().flat_map(extract_step).collect(),
221                }]
222            }
223        }
224        // Single section object
225        Value::Object(_) if is_how_to_section_obj(value) => {
226            extract_section(value).into_iter().collect()
227        }
228        // Single step or text
229        _ => vec![HowToSection {
230            name: None,
231            steps: extract_step(value),
232        }],
233    }
234}
235
236/// Extracts a [HowToSection]
237fn extract_section(value: &serde_json::Value) -> Option<HowToSection> {
238    if let Value::Object(obj) = value {
239        let name = obj
240            .get(LdFields::NAME)
241            .and_then(Value::as_str)
242            .map(|s| s.to_string());
243        let steps_val = obj.get(LdFields::ITEM_LIST_ELEMENT).unwrap_or(value);
244        Some(HowToSection {
245            name,
246            steps: extract_step(steps_val),
247        })
248    } else {
249        None
250    }
251}
252
253/// Extracts a [HowToStep]
254fn extract_step(value: &serde_json::Value) -> Vec<HowToStep> {
255    match value {
256        Value::Array(arr) => arr.iter().flat_map(extract_step).collect(),
257        Value::String(text) => vec![text.trim().to_string()],
258        Value::Object(obj) => obj
259            .get(LdFields::TEXT)
260            .and_then(Value::as_str)
261            .map(|text| vec![text.trim().to_string()])
262            .unwrap_or_default(),
263        _ => vec![],
264    }
265}
266
267/// Determines if the value is a [HowToSection] object.
268fn is_how_to_section_obj(value: &serde_json::Value) -> bool {
269    matches!(
270        value,
271        Value::Object(obj) if obj.get(LdFields::TYPE) == Some(&Value::String(HOW_TO_SECTION_TYPE.into()))
272    )
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278    use serde_json::json;
279
280    mod html_extraction {
281        use super::*;
282
283        #[test]
284        fn basic_1() {
285            let html = include_str!("../tests/fixtures/basic_1.html");
286            let recipe = parse_recipe(html).unwrap();
287            assert_eq!(
288                recipe,
289                Recipe {
290                    name: Some("recipe_name".into()),
291                    author: Some("author_name".into()),
292                    description: Some("description".into()),
293                    ingredients: vec!["ingredient_1".into(), "ingredient_2".into()],
294                    how_to_sections: vec![HowToSection {
295                        name: None,
296                        steps: vec!["instruction_1".into()],
297                    }]
298                }
299            )
300        }
301
302        #[test]
303        fn basic_2() {
304            let html = include_str!("../tests/fixtures/basic_2.html");
305            let recipe = parse_recipe(html).unwrap();
306            assert_eq!(
307                recipe,
308                Recipe {
309                    name: Some("recipe_name".into()),
310                    author: Some("author_name".into()),
311                    description: Some("description".into()),
312                    ingredients: vec!["ingredient_1".into(), "ingredient_2".into()],
313                    how_to_sections: vec![HowToSection {
314                        name: None,
315                        steps: vec!["instruction_1".into(), "instruction_2".into()],
316                    }]
317                }
318            )
319        }
320
321        #[test]
322        fn basic_3() {
323            let html = include_str!("../tests/fixtures/basic_3.html");
324            let recipe = parse_recipe(html).unwrap();
325            assert_eq!(
326                recipe,
327                Recipe {
328                    name: Some("recipe_name".into()),
329                    author: Some("author_name".into()),
330                    description: Some("description".into()),
331                    ingredients: vec!["ingredient_1".into(), "ingredient_2".into()],
332                    how_to_sections: vec![HowToSection {
333                        name: None,
334                        steps: vec!["instruction_1".into(), "instruction_2".into()],
335                    }]
336                }
337            )
338        }
339
340        #[test]
341        fn basic_4() {
342            let html = include_str!("../tests/fixtures/basic_4.html");
343            let recipe = parse_recipe(html).unwrap();
344            assert_eq!(
345                recipe,
346                Recipe {
347                    name: Some("recipe_name".into()),
348                    author: Some("author_name".into()),
349                    description: Some("description".into()),
350                    ingredients: vec!["ingredient_1".into(), "ingredient_2".into()],
351                    how_to_sections: vec![
352                        HowToSection {
353                            name: None,
354                            steps: vec!["instruction_1".into(), "instruction_2".into()],
355                        },
356                        HowToSection {
357                            name: Some("section_2".into()),
358                            steps: vec!["instruction_3".into(), "instruction_4".into()],
359                        }
360                    ]
361                }
362            )
363        }
364
365        #[test]
366        fn basic_5() {
367            let html = include_str!("../tests/fixtures/basic_5.html");
368            let recipe = parse_recipe(html).unwrap();
369            assert_eq!(
370                recipe,
371                Recipe {
372                    name: Some("recipe_name".into()),
373                    author: Some("author_name".into()),
374                    description: Some("description".into()),
375                    ingredients: vec!["ingredient_1".into(), "ingredient_2".into()],
376                    how_to_sections: vec![
377                        HowToSection {
378                            name: Some("section_1".into()),
379                            steps: vec!["instruction_1".into(), "instruction_2".into()],
380                        },
381                        HowToSection {
382                            name: Some("section_2".into()),
383                            steps: vec!["instruction_3".into(), "instruction_4".into()],
384                        }
385                    ]
386                }
387            )
388        }
389    }
390
391    mod name {
392        use super::*;
393
394        #[test]
395        fn extract_simple_name() {
396            let json = json!({"name": "Recipe Name"});
397            let recipe = extract_recipe(json.as_object().unwrap());
398            assert_eq!(recipe.name, Some("Recipe Name".to_string()));
399        }
400
401        #[test]
402        fn extract_missing_name() {
403            let json = json!({"description": "Recipe Name"});
404            let recipe = extract_recipe(json.as_object().unwrap());
405            assert_eq!(recipe.name, None);
406        }
407
408        #[test]
409        fn extract_non_string_name() {
410            let json = json!({"name": 123});
411            let recipe = extract_recipe(json.as_object().unwrap());
412            assert_eq!(recipe.name, None);
413        }
414    }
415
416    mod author {
417        use super::*;
418
419        #[test]
420        fn extract_string_author() {
421            let value = Value::String("John Doe".to_string());
422            let result = extract_author(&value);
423            assert_eq!(result, Some("John Doe".to_string()));
424        }
425
426        #[test]
427        fn extract_object_author() {
428            let value = json!({"name": "Jane Smith"});
429            let result = extract_author(&value);
430            assert_eq!(result, Some("Jane Smith".to_string()));
431        }
432
433        #[test]
434        fn extract_object_author_no_name() {
435            let value = json!({"email": "test@example.com"});
436            let result = extract_author(&value);
437            assert_eq!(result, None);
438        }
439
440        #[test]
441        fn extract_array_authors() {
442            let value = json!(["John Doe", "Jane Smith"]);
443            let result = extract_author(&value);
444            assert_eq!(result, Some("John Doe, Jane Smith".to_string()));
445        }
446
447        #[test]
448        fn extract_array_object_authors() {
449            let value = json!([
450                {"name": "John Doe"},
451                {"name": "Jane Smith"}
452            ]);
453            let result = extract_author(&value);
454            assert_eq!(result, Some("John Doe, Jane Smith".to_string()));
455        }
456
457        #[test]
458        fn extract_mixed_array_authors() {
459            let value = json!([
460                "John Doe",
461                {"name": "Jane Smith"},
462                {"email": "invalid@example.com"}
463            ]);
464            let result = extract_author(&value);
465            assert_eq!(result, Some("John Doe, Jane Smith".to_string()));
466        }
467
468        #[test]
469        fn extract_empty_array_authors() {
470            let value = json!([]);
471            let result = extract_author(&value);
472            assert_eq!(result, None);
473        }
474
475        #[test]
476        fn extract_invalid_type() {
477            let value = Value::Number(123.into());
478            let result = extract_author(&value);
479            assert_eq!(result, None);
480        }
481    }
482
483    mod description {
484        use super::*;
485
486        #[test]
487        fn extract_simple_description() {
488            let json = json!({"description": "A description"});
489            let recipe = extract_recipe(json.as_object().unwrap());
490            assert_eq!(recipe.description, Some("A description".to_string()));
491        }
492
493        #[test]
494        fn extract_missing_description() {
495            let json = json!({"name": "Cake"});
496            let recipe = extract_recipe(json.as_object().unwrap());
497            assert_eq!(recipe.description, None);
498        }
499
500        #[test]
501        fn extract_non_string_description() {
502            let json = json!({"description": 456});
503            let recipe = extract_recipe(json.as_object().unwrap());
504            assert_eq!(recipe.description, None);
505        }
506
507        #[test]
508        fn extract_empty_description() {
509            let json = json!({"description": ""});
510            let recipe = extract_recipe(json.as_object().unwrap());
511            assert_eq!(recipe.description, Some("".to_string()));
512        }
513    }
514
515    mod ingredients {
516        use super::*;
517
518        #[test]
519        fn extract_string_ingredient() {
520            let value = Value::String("1 cup flour".to_string());
521            let result = extract_ingredients(&value);
522            assert_eq!(result, vec!["1 cup flour"]);
523        }
524
525        #[test]
526        fn extract_array_ingredients() {
527            let value = json!(["1 cup flour", "2 eggs", "1 cup milk"]);
528            let result = extract_ingredients(&value);
529            assert_eq!(result, vec!["1 cup flour", "2 eggs", "1 cup milk"]);
530        }
531
532        #[test]
533        fn extract_mixed_array_ingredients() {
534            let value = json!(["1 cup flour", 123, "2 eggs"]);
535            let result = extract_ingredients(&value);
536            assert_eq!(result, vec!["1 cup flour", "2 eggs"]);
537        }
538
539        #[test]
540        fn extract_empty_array_ingredients() {
541            let value = json!([]);
542            let result = extract_ingredients(&value);
543            assert_eq!(result, Vec::<String>::new());
544        }
545
546        #[test]
547        fn extract_invalid_type_ingredients() {
548            let value = Value::Number(123.into());
549            let result = extract_ingredients(&value);
550            assert_eq!(result, Vec::<String>::new());
551        }
552
553        #[test]
554        fn extract_object_ingredients() {
555            let value = json!({"ingredient": "flour"});
556            let result = extract_ingredients(&value);
557            assert_eq!(result, Vec::<String>::new());
558        }
559
560        #[test]
561        fn extract_array_with_objects() {
562            let value = json!([
563                "1 cup flour",
564                {"name": "eggs"},
565                "2 tbsp sugar"
566            ]);
567            let result = extract_ingredients(&value);
568            assert_eq!(result, vec!["1 cup flour", "2 tbsp sugar"]);
569        }
570    }
571
572    mod instructions {
573        use super::*;
574
575        mod strings {
576            use super::*;
577
578            #[test]
579            fn extract_multiline() {
580                let instruction = r#"Step 1\nStep 2\nStep 3"#;
581                let value = Value::String(instruction.into());
582                let result = extract_instructions(&value);
583                assert_eq!(result.len(), 1);
584                assert_eq!(result[0].name, None);
585                assert_eq!(result[0].steps, vec![instruction]);
586            }
587
588            #[test]
589            fn extract_invalid_type() {
590                let result = extract_step(&Value::Number(1.into()));
591                assert_eq!(result, Vec::<String>::new());
592            }
593
594            #[test]
595            fn extract_array() {
596                let value = json!(["Step 1", "Step 2"]);
597                let result = extract_instructions(&value);
598                assert_eq!(result.len(), 1);
599                assert_eq!(result[0].name, None);
600                assert_eq!(result[0].steps, vec!["Step 1", "Step 2"]);
601            }
602        }
603
604        mod how_to_steps {
605            use super::*;
606
607            #[test]
608            fn extract_single() {
609                let value = json!({
610                    "text": "Do this",
611                });
612                let result = extract_instructions(&value);
613                assert_eq!(result.len(), 1);
614                assert_eq!(result[0].name, None);
615                assert_eq!(result[0].steps, vec!["Do this"]);
616            }
617
618            #[test]
619            fn extract_array() {
620                let value = json!([{
621                    "text": "Do this",
622                },{
623                    "text": "Do that"
624                }]);
625                let result = extract_instructions(&value);
626                assert_eq!(result.len(), 1);
627                assert_eq!(result[0].name, None);
628                assert_eq!(result[0].steps, vec!["Do this", "Do that"]);
629            }
630        }
631
632        mod how_to_sections {
633            use super::*;
634
635            #[test]
636            fn extract_single() {
637                let value = json!({
638                    "@type": "HowToSection",
639                    "name": "Test Section",
640                    "itemListElement": ["Step 1", "Step 2"]
641                });
642                let result = extract_section(&value);
643                assert!(result.is_some());
644                assert!(is_how_to_section_obj(&value));
645                let section = result.unwrap();
646                assert_eq!(section.name, Some("Test Section".to_string()));
647                assert_eq!(section.steps, vec!["Step 1", "Step 2"]);
648            }
649
650            #[test]
651            fn extract_single_invalid() {
652                let value = json!({
653                    "other": "stuff",
654                });
655                let result = extract_step(&value);
656                assert_eq!(result, Vec::<String>::new());
657            }
658
659            #[test]
660            fn extract_single_no_name() {
661                let value = json!({
662                    "@type": "HowToSection",
663                    "itemListElement": ["Step 1"]
664                });
665                let result = extract_section(&value);
666                assert!(is_how_to_section_obj(&value));
667                assert!(result.is_some());
668                let section = result.unwrap();
669                assert_eq!(section.name, None);
670                assert_eq!(section.steps, vec!["Step 1"]);
671            }
672
673            #[test]
674            fn extract_array() {
675                let value = json!([{
676                    "@type": "HowToSection",
677                    "name": "Preparation",
678                    "itemListElement": ["Preparation Step 1", "Preparation Step 2"]
679                },{
680                    "@type": "HowToSection",
681                    "name": "Cooking",
682                    "itemListElement": ["Cooking Step 1", "Cooking Step 2"]
683                }]);
684                let result = extract_instructions(&value);
685                assert_eq!(result.len(), 2);
686                assert_eq!(result[0].name, Some("Preparation".to_string()));
687                assert_eq!(
688                    result[0].steps,
689                    vec!["Preparation Step 1", "Preparation Step 2"]
690                );
691                assert_eq!(result[1].name, Some("Cooking".to_string()));
692                assert_eq!(result[1].steps, vec!["Cooking Step 1", "Cooking Step 2"]);
693            }
694
695            #[test]
696            fn extract_invalid() {
697                let result = extract_section(&Value::String("not an object".to_string()));
698                assert!(result.is_none());
699            }
700
701            #[test]
702            fn valid_section() {
703                let value = json!({
704                    "@type": "HowToSection"
705                });
706                assert!(is_how_to_section_obj(&value));
707            }
708
709            #[test]
710            fn invalid_section() {
711                let value = json!({
712                    "@type": "something"
713                });
714                assert!(!is_how_to_section_obj(&value));
715                let value = json!({
716                    "@type": "Recipe"
717                });
718                assert!(!is_how_to_section_obj(&value));
719                assert!(!is_how_to_section_obj(&Value::Object(Map::new())));
720                assert!(!is_how_to_section_obj(&Value::String("test".to_string())));
721            }
722        }
723    }
724}
reget/lib.rs

reget/
lib.rs
