scrapelect 0.3.2

Interpreter for scrapelect, a CSS-inspired web scraping DSL
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

import random

def scrapelect():
    return [[statement_list]]

def statement_list():
    return [[statement, statement_list]]

def statement():
    return [[randid(), ":", rvalue, filter_list, ";"]]

def rvalue():
    return [[element], [leaf]]

def element():
    return [[maybe_url, selector_list, selector_ops, "{", statement_list, "}"]]

def maybe_url():
    return [[variable], [randstr()]]

def selector_list():
    return [[selector, selector2]]

def selector():
    return [[".", randid()], ["#", randid()], [randid()], ["*"]]

def selector2():
    return [[selector_combinator, selector, selector2]]

def selector_combinator():
    # explicit whitespace here
    return [[x] for x in [" + ", " > ", " ~ ", ""]]

def selector_ops():
    return [["?"], ["[]"]]

def filter_list():
    return [["|", randid(), "(", arg_list, ")", filter_list]]

def arg_list():
    return [[randid(), ":", leaf, arg_list2]]

def arg_list2():
    return [[",", arg_list]]

def leaf():
    return [[variable], [randflt()], [randstr()], [randint()]]

def variable():
    return [["$", randid()]]

e = lambda: ""
def randid():
    return "".join(random.choice("abcdefghjiklmnopqrstuvwxyz") for _ in range(random.randint(1, 26)))

def randflt():
    return str((1 - random.random()) * 10000)

def randint():
    return str(random.randint(-10000, 10000))

def randstr():
    # doesn't test escaping or anything
    return f'"{randid()}"'

bases = {
    statement_list,
    maybe_url,
    selector2,
    selector_combinator,
    selector_ops,
    filter_list,
    arg_list,
    arg_list2
}

def fuzz(recurse_prob, f=scrapelect, a=None):
    if a is None: a = recurse_prob
    if f in bases:
        r = random.random()
        if r < 1 - recurse_prob:
            yield ""
            return

    rule = random.choice(f())
    for r in rule:
        if isinstance(r, str):
            yield r
        else:
            yield "".join(fuzz(recurse_prob * a, f=r))

if __name__ == '__main__':
    import sys
    r = float(sys.argv[1])
    for x in fuzz(r): print(x)