[package]
edition = "2021"
rust-version = "1.80"
name = "siteforge"
version = "0.1.19"
build = false
include = [
"Cargo.toml",
"Cargo.lock",
"README.md",
"CHANGELOG.md",
"config.example.yaml",
"src/**",
"tests/**",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Archive websites into AI-readable local knowledge archives"
homepage = "https://github.com/Tknott95/siteforge"
documentation = "https://docs.rs/siteforge"
readme = "README.md"
keywords = [
"archive",
"crawler",
"website",
"markdown",
"ai",
]
categories = [
"command-line-utilities",
"filesystem",
"web-programming",
]
license = "MIT OR Apache-2.0"
repository = "https://github.com/Tknott95/siteforge"
[lib]
name = "siteforge"
path = "src/lib.rs"
[[bin]]
name = "siteforge"
path = "src/main.rs"
[[test]]
name = "siteforge"
path = "tests/siteforge.rs"
[dependencies.aisling]
version = "0.3.17"
[dependencies.anyhow]
version = "1.0"
[dependencies.async-trait]
version = "0.1"
[dependencies.blake3]
version = "1.5"
[dependencies.chrono]
version = "0.4"
features = ["serde"]
[dependencies.clap]
version = "4.5"
features = [
"derive",
"env",
]
[dependencies.crossterm]
version = "0.28"
[dependencies.directories]
version = "5.0"
[dependencies.flate2]
version = "1.0"
[dependencies.futures-util]
version = "0.3"
[dependencies.globset]
version = "0.4"
[dependencies.html-escape]
version = "0.2"
[dependencies.imagesize]
version = "0.14"
[dependencies.once_cell]
version = "1.19"
[dependencies.regex]
version = "1.10"
[dependencies.reqwest]
version = "0.12"
features = [
"brotli",
"charset",
"deflate",
"gzip",
"http2",
"json",
"rustls-tls",
"stream",
]
default-features = false
[dependencies.robots_txt]
version = "0.7"
[dependencies.rusqlite]
version = "0.32"
features = [
"bundled",
"chrono",
]
[dependencies.scraper]
version = "0.20"
[dependencies.scrin]
version = "0.1.82"
[dependencies.serde]
version = "1.0"
features = ["derive"]
[dependencies.serde_json]
version = "1.0"
[dependencies.serde_yaml]
version = "0.9"
[dependencies.shell-words]
version = "1.1"
[dependencies.tar]
version = "0.4"
[dependencies.thiserror]
version = "1.0"
[dependencies.tokio]
version = "1.40"
features = [
"macros",
"rt-multi-thread",
"signal",
"time",
]
[dependencies.tracing]
version = "0.1"
[dependencies.tracing-subscriber]
version = "0.3"
features = [
"env-filter",
"fmt",
]
[dependencies.url]
version = "2.5"
features = ["serde"]
[dependencies.uuid]
version = "1.10"
features = [
"v4",
"serde",
]
[dependencies.walkdir]
version = "2.5"
[dependencies.zip]
version = "2.2"
features = ["deflate"]
default-features = false
[dev-dependencies.tempfile]
version = "3.10"
[dev-dependencies.wiremock]
version = "0.6"