nu_plugin_selector 0.44.0

web scraping using css selector
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use crate::{selector::begin_selector_query, Selector};
use nu_errors::ShellError;
use nu_plugin::Plugin;
use nu_protocol::{
    CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value,
};
use scraper::Selector as ScraperSelector;

impl Plugin for Selector {
    fn config(&mut self) -> Result<Signature, ShellError> {
        Ok(Signature::build("selector")
            .desc("execute selector query on html/web")
            .named("query", SyntaxShape::String, "selector query", Some('q'))
            .switch("as_html", "return the query output as html", Some('m'))
            .named(
                "attribute",
                SyntaxShape::String,
                "downselect based on the given attribute",
                Some('a'),
            )
            .named(
                "as_table",
                SyntaxShape::Table,
                "find table based on column header list",
                Some('t'),
            )
            .switch(
                "inspect",
                "run in inspect mode to provide more information for determining column headers",
                Some('i'),
            )
            .filter())
    }

    fn begin_filter(&mut self, call_info: CallInfo) -> Result<Vec<ReturnValue>, ShellError> {
        let tag = call_info.name_tag;
        // let query = call_info.args.nth(0).ok_or_else(|| {
        //     ShellError::labeled_error(
        //         "selector query not passed",
        //         "selector query not passed",
        //         &tag,
        //     )
        // })?;

        // self.query = query.as_string()?;
        self.query = if let Some(qtext) = call_info.args.get("query") {
            qtext.convert_to_string()
        } else {
            "".to_string()
        };
        self.tag = tag;
        self.as_html = call_info.args.has("as_html");
        if call_info.args.has("attribute") {
            self.attribute = call_info.args.expect_get("attribute")?.convert_to_string();
        }

        if call_info.args.has("as_table") {
            self.as_table = call_info.args.expect_get("as_table")?.clone();
        }
        self.inspect = call_info.args.has("inspect");

        Ok(vec![])
    }

    fn filter(&mut self, input: Value) -> Result<Vec<ReturnValue>, ShellError> {
        if !self.query.is_empty() && ScraperSelector::parse(&self.query).is_err() {
            return Err(ShellError::labeled_error(
                "Can not parse this query as a valid css selector",
                "Parse error",
                &self.tag,
            ));
        }
        match input {
            Value {
                value: UntaggedValue::Primitive(Primitive::String(s)),
                ..
            } => Ok(begin_selector_query(s, self)
                .into_iter()
                .map(ReturnSuccess::value)
                .collect()),
            Value { tag, .. } => Err(ShellError::labeled_error_with_secondary(
                "Expected text from pipeline",
                "requires text input",
                &self.tag,
                "value originates from here",
                tag,
            )),
        }
    }
}