urx 0.8.0

Extracts URLs from OSINT Archives for Security Insights.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

# URX Configuration File Example

# Domain configuration
domains = ["example.com", "example.org"]

# Output options
[output]
output = "results.txt"
format = "plain"       # Options: plain, json, csv
merge_endpoint = false

# Provider options
[provider]
providers = ["wayback", "cc", "otx"]
subs = false                         # Include subdomains when searching
cc_index = "CC-MAIN-2025-13"         # Common Crawl index to use
vt_api_key = ""                      # VirusTotal API key (If using VirusTotal)
exclude_robots = false               # Exclude robots.txt URLs
exclude_sitemap = false              # Exclude sitemap.xml URLs

# Display options
verbose = false
silent = false
no_progress = false

# Filter options
[filter]
preset = ["no-resources", "no-images"]  # Filter presets
extensions = ["js", "php", "aspx"]      # Include only these extensions
exclude_extensions = ["html", "txt"]    # Exclude these extensions
patterns = ["admin", "api"]             # Only include URLs with these patterns
exclude_patterns = ["logout", "static"] # Exclude URLs with these patterns
show_only_host = false
show_only_path = false
show_only_param = false
min_length = 10                         # Minimum URL length to include
max_length = 500                        # Maximum URL length to include
strict = true                           # Strict mode for host validation

# Network options
[network]
network_scope = "all"                   # Options: all, providers, testers, providers,testers
proxy = "http://proxy.example.com:8080" # HTTP proxy
proxy_auth = "username:password"        # Proxy authentication
insecure = false                        # Skip SSL certificate verification
random_agent = true                     # Use random User-Agent
timeout = 30                            # Request timeout in seconds
retries = 3                             # Number of retries for failed requests
parallel = 5                            # Maximum number of parallel requests
rate_limit = 10                         # Rate limit (requests per second)

# Testing options
[testing]
check_status = false            # Check HTTP status code of collected URLs
include_status = ["200", "30x"] # Include URLs with these status codes
exclude_status = ["404", "50x"] # Exclude URLs with these status codes
extract_links = false           # Extract additional links from collected URLs

# Cache options
[cache]
incremental = false             # Enable incremental scanning mode
cache_type = "sqlite"           # Cache backend: sqlite or redis
cache_path = "~/.urx/cache.db"  # Path for SQLite cache database
redis_url = "redis://localhost:6379"  # Redis connection URL (for redis cache_type)
cache_ttl = 86400              # Cache time-to-live in seconds (24 hours)
no_cache = false               # Disable caching entirely