synapse-waf 0.9.0

High-performance WAF and reverse proxy with embedded intelligence — built on Cloudflare Pingora
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

//! Known legitimate crawler definitions.

/// Definition of a legitimate crawler/bot.
#[derive(Debug, Clone)]
pub struct CrawlerDefinition {
    /// Crawler name
    pub name: &'static str,
    /// User agent regex pattern
    pub user_agent_pattern: &'static str,
    /// Reverse DNS regex pattern for verification
    pub reverse_dns_pattern: &'static str,
    /// Optional IP ranges (CIDR notation)
    pub ip_ranges: Option<&'static [&'static str]>,
    /// Whether DNS verification is required
    pub verification_required: bool,
}

/// Known legitimate crawlers.
pub static KNOWN_CRAWLERS: &[CrawlerDefinition] = &[
    // Major search engines (verification required)
    CrawlerDefinition {
        name: "Googlebot",
        user_agent_pattern: r"(?i)googlebot|google-inspectiontool|storebot-google",
        reverse_dns_pattern: r"(?i)\.(googlebot|google)\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "Bingbot",
        user_agent_pattern: r"(?i)bingbot|msnbot|bingpreview",
        reverse_dns_pattern: r"(?i)\.search\.msn\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "Baiduspider",
        user_agent_pattern: r"(?i)baiduspider",
        reverse_dns_pattern: r"(?i)\.crawl\.baidu\.(com|jp)$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "YandexBot",
        user_agent_pattern: r"(?i)yandexbot",
        reverse_dns_pattern: r"(?i)\.yandex\.(com|ru|net)$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "DuckDuckBot",
        user_agent_pattern: r"(?i)duckduckbot",
        reverse_dns_pattern: r"(?i)\.duckduckgo\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "Slurp",
        user_agent_pattern: r"(?i)slurp",
        reverse_dns_pattern: r"(?i)\.crawl\.yahoo\.net$",
        ip_ranges: None,
        verification_required: true,
    },
    // Social media crawlers (verification required)
    CrawlerDefinition {
        name: "Facebookbot",
        user_agent_pattern: r"(?i)facebookexternalhit",
        reverse_dns_pattern: r"(?i)\.(facebook|fbsv)\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "Twitterbot",
        user_agent_pattern: r"(?i)twitterbot",
        reverse_dns_pattern: r"(?i)\.twitter\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "LinkedInBot",
        user_agent_pattern: r"(?i)linkedinbot",
        reverse_dns_pattern: r"(?i)\.linkedin\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    CrawlerDefinition {
        name: "Applebot",
        user_agent_pattern: r"(?i)applebot",
        reverse_dns_pattern: r"(?i)\.applebot\.apple\.com$",
        ip_ranges: None,
        verification_required: true,
    },
    // SEO tools (verification not required - trust UA)
    CrawlerDefinition {
        name: "AhrefsBot",
        user_agent_pattern: r"(?i)ahrefsbot",
        reverse_dns_pattern: r"(?i)\.ahrefs\.com$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "SemrushBot",
        user_agent_pattern: r"(?i)semrushbot",
        reverse_dns_pattern: r"(?i)\.semrush\.com$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "MJ12bot",
        user_agent_pattern: r"(?i)mj12bot",
        reverse_dns_pattern: r"(?i)\.majestic12\.co\.uk$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "DotBot",
        user_agent_pattern: r"(?i)dotbot",
        reverse_dns_pattern: r"(?i)\.opensiteexplorer\.com$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "ScreamingFrog",
        user_agent_pattern: r"(?i)screaming frog seo spider",
        reverse_dns_pattern: r".*", // No DNS verification
        ip_ranges: None,
        verification_required: false,
    },
    // Communication platform bots
    CrawlerDefinition {
        name: "Pinterestbot",
        user_agent_pattern: r"(?i)pinterest",
        reverse_dns_pattern: r"(?i)\.pinterest\.com$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "Slackbot",
        user_agent_pattern: r"(?i)slackbot|slackbot-linkexpanding",
        reverse_dns_pattern: r"(?i)\.slack\.com$",
        ip_ranges: None,
        verification_required: false,
    },
    CrawlerDefinition {
        name: "Discordbot",
        user_agent_pattern: r"(?i)discordbot",
        reverse_dns_pattern: r"(?i)\.discord\.com$",
        ip_ranges: None,
        verification_required: false,
    },
];