stygian-plugin 0.14.0

Visual data extraction fallback subsystem with CSS/XPath selectors, idempotent request handling, and composable transformation pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

#![cfg(feature = "graph-integration")]
#![allow(
    clippy::panic,
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing,
    clippy::missing_const_for_fn
)]
//! Integration test for plugin extraction

#[cfg(test)]
mod tests {
    use serde_json::json;
    use std::sync::Arc;
    use stygian_graph::ports::{ScrapingService, ServiceInput};
    use stygian_plugin::{
        adapters::{ExtractionEngine, PluginExtractionAdapter},
        domain::{ExtractionTemplate, Region, Selector, Transformation},
        ports::PluginTemplateStore,
        storage::{FileTemplateStore, MemoryIdempotencyStore},
    };
    use tempfile::TempDir;

    #[tokio::test]
    async fn test_complete_extraction_workflow()
    -> std::result::Result<(), Box<dyn std::error::Error>> {
        // Create storage
        let tmp = TempDir::new()?;
        let template_store = Arc::new(FileTemplateStore::new(tmp.path().to_path_buf()));
        let extraction_port = Arc::new(ExtractionEngine);
        let idempotency_store = Arc::new(MemoryIdempotencyStore::new());

        // Create template
        let template = ExtractionTemplate::new("Product Listing")
            .with_description("Extract product information")
            .with_region(
                Region::new(
                    "product_name",
                    Selector::css(".product-name"),
                    json!({"type": "string"}),
                )
                .with_transformation(Transformation::Trim)
                .with_transformation(Transformation::NormalizeWhitespace),
            )
            .with_region(
                Region::new(
                    "price",
                    Selector::css(".product-price"),
                    json!({"type": "string"}),
                )
                .with_transformation(Transformation::Trim),
            );

        // Save template
        template_store.save(&template).await?;
        let template_id = template.id;

        // Create adapter
        let adapter =
            PluginExtractionAdapter::new(template_store, extraction_port, idempotency_store);

        // Sample HTML for extraction
        let html = r#"
            <div class="product-name">Premium Widget</div>
            <span class="product-price">$19.99</span>
        "#;

        let input = ServiceInput {
            url: "https://example.com/products".to_string(),
            params: json!({
                "template_id": template_id.to_string(),
                "html": html,
            }),
        };

        let result = adapter.execute(input).await?;

        // Verify output
        assert!(!result.data.is_empty());
        assert_eq!(result.metadata.get("cached"), Some(&json!(false)));

        // Parse extracted data
        let extracted: serde_json::Value =
            serde_json::from_str(&result.data).unwrap_or_else(|_| json!({}));
        assert!(!extracted.is_null());
        Ok(())
    }

    #[tokio::test]
    async fn test_idempotency_deduplication() -> std::result::Result<(), Box<dyn std::error::Error>>
    {
        let tmp = TempDir::new()?;
        let template_store = Arc::new(FileTemplateStore::new(tmp.path().to_path_buf()));
        let extraction_port = Arc::new(ExtractionEngine);
        let idempotency_store = Arc::new(MemoryIdempotencyStore::new());

        // Create and save template
        let template = ExtractionTemplate::new("Test").with_region(Region::new(
            "title",
            Selector::css("h1"),
            json!({"type": "string"}),
        ));
        template_store.save(&template).await?;

        let adapter =
            PluginExtractionAdapter::new(template_store, extraction_port, idempotency_store);

        let idempotency_key = ulid::Ulid::new();
        let html = "<html><h1>First</h1></html>";

        let input1 = ServiceInput {
            url: "https://example.com/page1".to_string(),
            params: json!({
                "template_id": template.id.to_string(),
                "idempotency_key": idempotency_key.to_string(),
                "html": html,
            }),
        };

        let result1 = adapter.execute(input1).await?;
        assert_eq!(result1.metadata.get("cached"), Some(&json!(false)));

        // Same key with different URL/HTML should return cached result (idempotency key is primary)
        let input2 = ServiceInput {
            url: "https://example.com/page2".to_string(),
            params: json!({
                "template_id": template.id.to_string(),
                "idempotency_key": idempotency_key.to_string(),
                "html": "<html><h1>Different</h1></html>",
            }),
        };

        let result2 = adapter.execute(input2).await?;
        assert_eq!(result2.metadata.get("cached"), Some(&json!(true)));
        assert_eq!(result1.data, result2.data);
        Ok(())
    }
}