scrape-le 0.1.0

Check whether a page is scrapeable before the scraper is written
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

{
	"validate": [
		{ "input": "https://example.com", "expected": true },
		{ "input": "http://example.com", "expected": true },
		{ "input": "HTTPS://EXAMPLE.COM", "expected": true },
		{ "input": "  https://example.com  ", "expected": true },
		{ "input": "http://localhost:3000", "expected": true },
		{ "input": "https://192.168.1.1/admin", "expected": true },
		{
			"input": "https://sub.example.co.uk/path?q=1#frag",
			"expected": true
		},
		{ "input": "ftp://example.com", "expected": false },
		{ "input": "example.com", "expected": false },
		{ "input": "https://", "expected": false },
		{ "input": "not a url", "expected": false },
		{ "input": "", "expected": false }
	],
	"normalize": [
		{ "input": "example.com", "expected": "https://example.com" },
		{ "input": "http://example.com", "expected": "http://example.com" },
		{
			"input": "  www.example.com  ",
			"expected": "https://www.example.com"
		},
		{
			"input": "  https://example.com  ",
			"expected": "https://example.com"
		},
		{
			"input": "ftp://example.com",
			"expected": "https://ftp://example.com"
		}
	],
	"extract": [
		{
			"input": "Check https://example.com for details",
			"expected": "https://example.com"
		},
		{
			"input": "https://a.com and https://b.com",
			"expected": "https://a.com"
		},
		{ "input": "example.com", "expected": "https://example.com" },
		{ "input": "no url here", "expected": null },
		{ "input": "", "expected": null },
		{ "input": "Visit www.example.com today", "expected": null },
		{
			"input": "(https://en.wikipedia.org/wiki/Rust_(programming_language))",
			"expected": "https://en.wikipedia.org/wiki/Rust_(programming_language))"
		},
		{
			"input": "trailing dot https://example.com.",
			"expected": "https://example.com."
		}
	]
}