thesa 4.1.34

Archive GitHub repositories, ML models, datasets, and websites with Scrin/Aisling workflows
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
use std::path::PathBuf;

use clap::{Args, Parser, Subcommand};
use siteforge::ExportFormat as SiteforgeExportFormat;

use crate::types::{DatasetProvider, Mode, ModelProvider, SiteArchiveProfile};

#[derive(Parser, Clone)]
#[command(
    name = "thesa",
    version,
    about = "A compendium to archive GitHub repositories, ML models, datasets, and websites",
    subcommand_precedence_over_arg = true
)]
pub(crate) struct Cli {
    #[command(subcommand)]
    pub(crate) command: Option<Commands>,

    /// Target: GitHub owner/repo (repos), model target, dataset target, or website URL
    pub(crate) target: Option<String>,

    /// Output directory where archives are stored
    #[arg(short = 'o', long = "output", default_value = "./archives")]
    pub(crate) output: PathBuf,

    /// GitHub token for API requests (optional, falls back to GITHUB_TOKEN env)
    #[arg(short = 't', long = "token")]
    pub(crate) token: Option<String>,

    /// Hugging Face token for API requests (optional, falls back to HF_TOKEN env)
    #[arg(long = "hf-token")]
    pub(crate) hf_token: Option<String>,

    /// CivitAI token for API requests (optional, falls back to CIVITAI_TOKEN env)
    #[arg(long = "civitai-token")]
    pub(crate) civitai_token: Option<String>,

    /// Clone depth for each repository
    #[arg(long = "depth")]
    pub(crate) depth: Option<u32>,

    /// Number of items to process at once
    #[arg(short = 'c', long = "concurrency", default_value_t = 4)]
    pub(crate) concurrency: usize,

    /// Skip existing directories instead of failing or replacing
    #[arg(long = "skip-existing")]
    pub(crate) skip_existing: bool,

    /// Filter items by substring in name
    #[arg(long = "filter")]
    pub(crate) filter: Option<String>,

    /// Print plan only; do not execute
    #[arg(long = "dry-run")]
    pub(crate) dry_run: bool,

    /// Force the interactive Scrin interface
    #[arg(short = 'T', long = "tui")]
    pub(crate) tui: bool,

    /// Archive mode: repos (GitHub), models, datasets, or sites (Siteforge)
    #[arg(
        long = "mode",
        value_enum,
        default_value_t = Mode::Repos,
        help = "Archive mode: repos (GitHub), models, datasets, or sites (Siteforge)"
    )]
    pub(crate) mode: Mode,

    /// Model provider: hf (Hugging Face), ollama, civitai
    #[arg(long = "provider", value_enum, default_value_t = ModelProvider::Hf)]
    pub(crate) provider: ModelProvider,

    /// Dataset provider: hf (Hugging Face Datasets), zenodo
    #[arg(long = "dataset-provider", value_enum, default_value_t = DatasetProvider::Hf)]
    pub(crate) dataset_provider: DatasetProvider,

    /// Use Hugging Face mirror (hf-mirror.com) — no token required
    #[arg(long = "hf-mirror")]
    pub(crate) hf_mirror: bool,
}

#[derive(Debug, Clone, Subcommand)]
#[allow(clippy::large_enum_variant)]
pub(crate) enum Commands {
    /// Archive artifacts from a supported source provider
    Archive {
        /// Print plan only; do not execute
        #[arg(long = "dry-run")]
        dry_run: bool,

        #[command(subcommand)]
        source: ArchiveCommands,
    },

    /// Verify a thesa archive directory using .thesa/manifest.json and checksums
    Verify {
        /// Archive output directory containing a .thesa manifest sidecar
        archive: PathBuf,

        /// Emit machine-readable JSON report
        #[arg(long)]
        json: bool,
    },
}

#[derive(Debug, Clone, Subcommand)]
pub(crate) enum ArchiveCommands {
    /// Refresh existing GitHub repository archives with git pull --ff-only
    #[command(alias = "update", alias = "pull")]
    Refresh(GitRefreshArgs),

    /// Archive a website with siteforge
    Site(SiteArchiveArgs),

    /// Archive datasets with baseforge
    Dataset(DatasetArchiveArgs),

    /// Archive a generic URL/website with siteforge
    Url(SiteArchiveArgs),

    /// List Siteforge website archives
    List(SiteforgeRootArgs),

    /// Inspect one Siteforge archive
    Inspect(SiteforgeArchiveIdArgs),

    /// Search Siteforge archives
    Search(SiteforgeSearchArgs),

    /// Export a Siteforge archive
    Export(SiteforgeExportArgs),

    /// Create AI context packs for a Siteforge archive
    Pack(SiteforgePackArgs),

    /// Verify a Siteforge archive by archive id
    Verify(SiteforgeArchiveIdArgs),

    /// Resume an interrupted Siteforge archive by archive id
    Resume(SiteforgeArchiveIdArgs),

    /// Print Siteforge diagnostics
    Diagnostics(SiteforgeDiagnosticsArgs),
}

#[derive(Debug, Clone, Args)]
pub(crate) struct GitRefreshArgs {
    /// Optional GitHub owner/user/org to refresh from the archive tree
    #[arg(value_name = "OWNER")]
    pub(crate) owner: Option<String>,

    /// Root to scan for GitHub repository archives
    #[arg(long = "archive-root", short = 'r', default_value = "./archives")]
    pub(crate) archive_root: PathBuf,

    /// GitHub owner/user/org to refresh from the archive tree
    #[arg(
        long = "owner",
        alias = "user",
        alias = "username",
        value_name = "OWNER"
    )]
    pub(crate) owner_filter: Option<String>,

    /// Number of repositories to refresh at once
    #[arg(short = 'c', long = "concurrency", default_value_t = 4)]
    pub(crate) concurrency: usize,

    /// Only refresh archives whose owner/repo slug or path contains this text
    #[arg(long = "filter")]
    pub(crate) filter: Option<String>,

    /// Print discovered archives only; do not run git pull
    #[arg(long = "dry-run")]
    pub(crate) dry_run: bool,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeRootArgs {
    /// Siteforge archive root
    #[arg(long = "archive-root", default_value = "./archives/sites")]
    pub(crate) archive_root: PathBuf,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct DatasetArchiveArgs {
    /// Dataset target: namespace, dataset id, `search:<query>`, top, latest, or provider URL
    pub(crate) target: String,

    /// Dataset provider
    #[arg(long = "provider", value_enum, default_value_t = DatasetProvider::Hf)]
    pub(crate) provider: DatasetProvider,

    /// Dataset archive root
    #[arg(
        long = "archive-root",
        short = 'r',
        default_value = "./archives/datasets"
    )]
    pub(crate) archive_root: PathBuf,

    /// Number of dataset files to download at once
    #[arg(short = 'c', long = "concurrency", default_value_t = 4)]
    pub(crate) concurrency: usize,

    /// Skip existing dataset directories
    #[arg(long = "skip-existing")]
    pub(crate) skip_existing: bool,

    /// Filter discovered datasets by substring
    #[arg(long = "filter")]
    pub(crate) filter: Option<String>,

    /// Hugging Face token for private/gated datasets (falls back to HF_TOKEN env)
    #[arg(long = "hf-token")]
    pub(crate) hf_token: Option<String>,

    /// Print plan only; do not download
    #[arg(long = "dry-run")]
    pub(crate) dry_run: bool,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeArchiveIdArgs {
    /// Siteforge archive id
    pub(crate) archive_id: String,

    #[command(flatten)]
    pub(crate) root: SiteforgeRootArgs,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeSearchArgs {
    /// Query text
    pub(crate) query: String,

    /// Restrict search to one archive id
    #[arg(long = "archive-id")]
    pub(crate) archive_id: Option<String>,

    /// Maximum matches to print
    #[arg(long = "limit", default_value_t = 20)]
    pub(crate) limit: usize,

    #[command(flatten)]
    pub(crate) root: SiteforgeRootArgs,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeExportArgs {
    /// Siteforge archive id
    pub(crate) archive_id: String,

    /// Export format
    #[arg(long = "format", value_enum)]
    pub(crate) format: SiteforgeExportFormat,

    /// Export output path
    #[arg(long = "output")]
    pub(crate) output: Option<PathBuf>,

    #[command(flatten)]
    pub(crate) root: SiteforgeRootArgs,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgePackArgs {
    /// Siteforge archive id
    pub(crate) archive_id: String,

    /// Maximum tokens per generated pack
    #[arg(long = "max-tokens")]
    pub(crate) max_tokens: Option<usize>,

    #[command(flatten)]
    pub(crate) root: SiteforgeRootArgs,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeDiagnosticsArgs {
    /// Optional Siteforge archive id
    pub(crate) archive_id: Option<String>,

    #[command(flatten)]
    pub(crate) root: SiteforgeRootArgs,
}

#[derive(Debug, Clone, Args)]
pub(crate) struct SiteArchiveArgs {
    /// Seed URL to archive. Optional when --input provides seeds.
    #[arg(value_name = "URL")]
    pub(crate) url: Option<String>,

    /// File containing seed URLs, one per line. Blank lines and # comments are ignored.
    #[arg(long = "input", value_name = "FILE")]
    pub(crate) input: Option<PathBuf>,

    /// Output root for website archives
    #[arg(short = 'o', long = "output", default_value = "./archives/sites")]
    pub(crate) output: PathBuf,

    /// Archive all in-scope pages under the site instead of one page
    #[arg(long = "full-site")]
    pub(crate) full_site: bool,

    /// Allow following links outside the seed domain
    #[arg(long = "allow-cross-domain")]
    pub(crate) allow_cross_domain: bool,

    /// Maximum crawl depth
    #[arg(long = "max-depth")]
    pub(crate) max_depth: Option<usize>,

    /// Maximum pages to archive
    #[arg(long = "max-pages")]
    pub(crate) max_pages: Option<usize>,

    /// Number of concurrent fetch workers
    #[arg(short = 'c', short_alias = 'j', long = "concurrency", alias = "jobs")]
    pub(crate) concurrency: Option<usize>,

    /// Politeness delay between requests per worker, in milliseconds
    #[arg(long = "delay-ms")]
    pub(crate) delay_ms: Option<u64>,

    /// Stable archive id for the siteforge archive
    #[arg(long = "archive-id")]
    pub(crate) archive_id: Option<String>,

    /// Include URL glob pattern. May be repeated.
    #[arg(long = "include", value_name = "GLOB")]
    pub(crate) include_url_patterns: Vec<String>,

    /// Exclude URL glob pattern. May be repeated.
    #[arg(long = "exclude", value_name = "GLOB")]
    pub(crate) exclude_url_patterns: Vec<String>,

    /// Siteforge archive output profile: full, agent, or lean
    #[arg(long = "archive-profile", value_enum)]
    pub(crate) archive_profile: Option<SiteArchiveProfile>,

    /// Maximum bytes to fetch per HTML page
    #[arg(long = "max-page-size-bytes")]
    pub(crate) max_page_size_bytes: Option<u64>,

    /// Maximum bytes to fetch per asset
    #[arg(long = "max-asset-size-bytes")]
    pub(crate) max_asset_size_bytes: Option<u64>,

    /// Maximum total archive size in bytes
    #[arg(long = "max-total-archive-size-bytes")]
    pub(crate) max_total_archive_size_bytes: Option<u64>,

    /// Render JavaScript pages through siteforge's browser renderer
    #[arg(long = "render-js")]
    pub(crate) render_js: bool,

    /// Render only pages that look like thin JavaScript app shells
    #[arg(long = "render-js-auto", conflicts_with = "render_js")]
    pub(crate) render_js_auto: bool,

    /// Browser command template for Siteforge JavaScript rendering
    #[arg(long = "browser-command", value_name = "COMMAND")]
    pub(crate) browser_command: Option<String>,

    /// Browser profile directory for Siteforge JavaScript rendering
    #[arg(long = "browser-profile", value_name = "DIR")]
    pub(crate) browser_profile: Option<PathBuf>,

    /// Milliseconds to wait after browser render before extracting DOM
    #[arg(long = "render-wait-ms")]
    pub(crate) render_wait_ms: Option<u64>,

    /// Extra request header. Repeatable. Format: "Name: Value"
    #[arg(long = "header", value_name = "NAME: VALUE")]
    pub(crate) headers: Vec<String>,

    /// Cookie header value to send with requests
    #[arg(long = "cookie", value_name = "COOKIE_HEADER")]
    pub(crate) cookie: Option<String>,

    /// User-Agent header used by Siteforge fetchers/renderers
    #[arg(long = "user-agent", value_name = "VALUE")]
    pub(crate) user_agent: Option<String>,

    /// HTTP timeout per request, in seconds
    #[arg(long = "timeout-secs")]
    pub(crate) timeout_secs: Option<u64>,

    /// Number of Siteforge retry attempts for failed frontier items
    #[arg(long = "retry-count")]
    pub(crate) retry_count: Option<usize>,

    /// Enable Siteforge OCR for code-like downloaded assets
    #[arg(long = "ocr")]
    pub(crate) ocr_enabled: bool,

    /// Print plan only; do not execute
    #[arg(long = "dry-run")]
    pub(crate) dry_run: bool,
}