siteforge 0.1.19

Archive websites into AI-readable local knowledge archives
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

[package]
name = "siteforge"
version = "0.1.19"
edition = "2021"
rust-version = "1.80"
description = "Archive websites into AI-readable local knowledge archives"
license = "MIT OR Apache-2.0"
readme = "README.md"
repository = "https://github.com/Tknott95/siteforge"
homepage = "https://github.com/Tknott95/siteforge"
documentation = "https://docs.rs/siteforge"
keywords = ["archive", "crawler", "website", "markdown", "ai"]
categories = ["command-line-utilities", "filesystem", "web-programming"]
include = ["Cargo.toml", "Cargo.lock", "README.md", "CHANGELOG.md", "config.example.yaml", "src/**", "tests/**"]

[dependencies]
aisling = "0.3.17"
anyhow = "1.0"
async-trait = "0.1"
blake3 = "1.5"
chrono = { version = "0.4", features = ["serde"] }
clap = { version = "4.5", features = ["derive", "env"] }
crossterm = "0.28"
directories = "5.0"
flate2 = "1.0"
futures-util = "0.3"
globset = "0.4"
html-escape = "0.2"
imagesize = "0.14"
once_cell = "1.19"
regex = "1.10"
reqwest = { version = "0.12", default-features = false, features = ["brotli", "charset", "deflate", "gzip", "http2", "json", "rustls-tls", "stream"] }
robots_txt = "0.7"
rusqlite = { version = "0.32", features = ["bundled", "chrono"] }
scraper = "0.20"
scrin = "0.1.82"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde_yaml = "0.9"
shell-words = "1.1"
tar = "0.4"
thiserror = "1.0"
tokio = { version = "1.40", features = ["macros", "rt-multi-thread", "signal", "time"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
url = { version = "2.5", features = ["serde"] }
uuid = { version = "1.10", features = ["v4", "serde"] }
walkdir = "2.5"
zip = { version = "2.2", default-features = false, features = ["deflate"] }

[dev-dependencies]
tempfile = "3.10"
wiremock = "0.6"