sqlite_graphrag/commands/ingest/
args.rs

1//! CLI arguments and low-memory / parallelism resolution for `ingest`.
2
3use crate::cli::MemoryType;
4use crate::output::JsonOutputFormat;
5use std::path::PathBuf;
6
7#[derive(clap::Args)]
8#[command(after_long_help = "EXAMPLES:\n  \
9    # Ingest every Markdown file under ./docs as `document` memories\n  \
10    sqlite-graphrag ingest ./docs --type document\n\n  \
11    # Ingest .txt files recursively under ./notes\n  \
12    sqlite-graphrag ingest ./notes --type note --pattern '*.txt' --recursive\n\n  \
13    # Namespace derived names with a kebab-case prefix (projx-<derived>)\n  \
14    sqlite-graphrag ingest ./docs --name-prefix projx- --dry-run\n\n  \
15    # Enable automatic URL extraction (URL-regex only since v1.0.79)\n  \
16    sqlite-graphrag ingest ./big-corpus --type reference --enable-ner\n\n  \
17    # Preview file-to-name mapping without ingesting\n  \
18    sqlite-graphrag ingest ./docs --dry-run\n\n  \
19    # LLM-curated extraction via Claude Code CLI\n  \
20    sqlite-graphrag ingest ./docs --mode claude-code --recursive --json\n\n  \
21    # Resume interrupted claude-code ingest\n  \
22    sqlite-graphrag ingest ./docs --mode claude-code --resume --json\n\n  \
23    # Claude Code with budget cap and custom timeout\n  \
24    sqlite-graphrag ingest ./docs --mode claude-code --max-cost-usd 5.00 --claude-timeout 600 --json\n\n  \
25AUTHENTICATION:\n  \
26    --mode claude-code: Uses existing Claude Code authentication.\n  \
27      OAuth (Pro/Max/Team): works automatically from ~/.claude/.credentials.json\n  \
28      API key: set ANTHROPIC_API_KEY for faster startup (optional)\n\n  \
29    --mode codex: Uses existing Codex CLI authentication.\n  \
30      Device auth: run `codex auth login` first\n  \
31      API key: set OPENAI_API_KEY (optional)\n\n  \
32NOTES:\n  \
33    Each file becomes a separate memory. Names derive from file basenames\n  \
34    (kebab-case, lowercase, ASCII). Output is NDJSON: one JSON object per file,\n  \
35    followed by a final summary line with counts. Per-file errors are reported\n  \
36    inline and processing continues unless --fail-fast is set.")]
37/// Ingest args.
38pub struct IngestArgs {
39    /// Directory containing files to ingest.
40    #[arg(
41        value_name = "DIR",
42        help = "Directory to ingest recursively (each matching file becomes a memory)"
43    )]
44    pub dir: PathBuf,
45
46    /// Memory type stored in `memories.type` for every ingested file. Defaults to `document`.
47    #[arg(long, value_enum, default_value_t = MemoryType::Document)]
48    pub r#type: MemoryType,
49
50    /// Glob pattern matched against file basenames (default: `*.md`). Supports
51    /// `*.<ext>`, `<prefix>*`, and exact filename match.
52    #[arg(long, default_value = "*.md")]
53    pub pattern: String,
54
55    /// Recurse into subdirectories.
56    #[arg(long, default_value_t = false)]
57    pub recursive: bool,
58
59    #[arg(
60        long,
61                value_parser = crate::parsers::parse_bool_flexible,
62        action = clap::ArgAction::Set,
63        num_args = 0..=1,
64        default_missing_value = "true",
65        default_value = "false",
66        help = "Enable automatic URL-regex extraction (URL-regex only since v1.0.79)"
67    )]
68    /// Enable NER.
69    pub enable_ner: bool,
70
71    /// GAP-E2E-011: generates a heuristic description from the first meaningful
72    /// line of the body, instead of "ingested from `<path>`". When
73    /// `--no-auto-describe` is passed, keeps the legacy behaviour.
74    #[arg(
75        long,
76        default_value_t = true,
77        overrides_with = "no_auto_describe",
78        help = "Derive memory description from the first meaningful body line instead of the legacy `ingested from <path>` placeholder."
79    )]
80    pub auto_describe: bool,
81    #[arg(
82        long = "no-auto-describe",
83        default_value_t = false,
84        help = "Disable `--auto-describe` and fall back to the legacy `ingested from <path>` description placeholder."
85    )]
86    /// No auto describe.
87    pub no_auto_describe: bool,
88
89    /// Deprecated: NER is now disabled by default. Kept for backwards compatibility.
90    #[arg(long, default_value_t = false, hide = true)]
91    pub skip_extraction: bool,
92
93    /// Stop on first per-file error instead of continuing with the next file.
94    #[arg(long, default_value_t = false)]
95    pub fail_fast: bool,
96
97    /// Preview file-to-name mapping without loading model or persisting.
98    #[arg(long, default_value_t = false)]
99    pub dry_run: bool,
100
101    /// Maximum number of files to ingest (safety cap to prevent runaway ingestion).
102    #[arg(long, default_value_t = 10_000)]
103    pub max_files: usize,
104
105    /// Namespace for the ingested memories.
106    #[arg(long)]
107    pub namespace: Option<String>,
108
109    /// Database path. Falls back to XDG `db.path`, then `graphrag.sqlite`
110    /// under the XDG data directory.
111    #[arg(long)]
112    pub db: Option<String>,
113
114    /// Output format.
115    #[arg(long, value_enum, default_value_t = JsonOutputFormat::Json)]
116    pub format: JsonOutputFormat,
117
118    /// Emit machine-readable JSON on stdout.
119    #[arg(long, hide = true, help = "No-op; JSON is always emitted on stdout")]
120    pub json: bool,
121
122    /// Number of files to extract+embed in parallel; default = max(1, cpus/2).min(4).
123    #[arg(
124        long,
125        help = "Number of files to extract+embed in parallel; default = max(1, cpus/2).min(4)"
126    )]
127    pub ingest_parallelism: Option<usize>,
128
129    /// Force single-threaded ingest to reduce RSS pressure.
130    ///
131    /// Equivalent to `--ingest-parallelism 1`, takes precedence over any
132    /// explicit value. Recommended for environments with <4 GB available
133    /// RAM or container/cgroup constraints. Trade-off: 3-4x longer wall
134    /// time. Also available via XDG `ingest.low_memory=1`
135    /// (CLI flag has higher precedence than the env var).
136    #[arg(
137        long,
138        default_value_t = false,
139        help = "Forces single-threaded ingest (--ingest-parallelism 1) to reduce RSS pressure. \
140                Recommended for environments with <4 GB available RAM or container/cgroup \
141                constraints. Trade-off: 3-4x longer wall time. Also honored via \
142                XDG ingest.low_memory=1."
143    )]
144    pub low_memory: bool,
145
146    /// Maximum process RSS in MiB; abort if exceeded during embedding.
147    #[arg(long, default_value_t = crate::constants::DEFAULT_MAX_RSS_MB,
148          help = "Maximum process RSS in MiB; abort if exceeded during embedding (default: 8192)")]
149    pub max_rss_mb: u64,
150
151    /// G42/S3 (v1.0.79): maximum simultaneous LLM embedding subprocesses
152    /// PER FILE. Multiplies with --ingest-parallelism (files staged
153    /// concurrently), hence the conservative default of 2. The effective
154    /// value is further bounded by CPU count and available RAM.
155    #[arg(long, default_value_t = 2, value_name = "N",
156          value_parser = clap::value_parser!(u64).range(1..=32),
157          help = "Maximum simultaneous LLM embedding subprocesses per file (default: 2, clamp [1,32])")]
158    pub llm_parallelism: u64,
159
160    /// Maximum character length for derived memory names from file basenames.
161    ///
162    /// Overrides the compile-time `DERIVED_NAME_MAX_LEN` constant (default 60).
163    /// Shorter values leave more headroom for collision suffix resolution.
164    #[arg(long, default_value_t = crate::constants::DERIVED_NAME_MAX_LEN,
165          help = "Maximum length for derived memory names (default: 60)")]
166    pub max_name_length: usize,
167
168    /// v1.1.1 (P12): kebab-case prefix prepended to every derived memory name,
169    /// AFTER the basename is normalized. Namespaces a corpus inside a shared
170    /// database (e.g. `--name-prefix projx-` yields `projx-<derived>`). The
171    /// derived part's budget shrinks so the final name always respects the
172    /// 80-char name cap. Only supported with `--mode none`.
173    #[arg(
174        long,
175        value_name = "PREFIX",
176        help = "Kebab-case prefix applied to every derived memory name (e.g. 'projx-')"
177    )]
178    pub name_prefix: Option<String>,
179
180    /// Extraction mode: `none` (body-only, default) or `claude-code`/`codex`/`opencode` (LLM-curated).
181    #[arg(long, value_enum, default_value_t = IngestMode::None)]
182    pub mode: IngestMode,
183
184    /// Explicit path to the Claude Code binary (only with --mode claude-code).
185    #[arg(long)]
186    pub claude_binary: Option<std::path::PathBuf>,
187
188    /// Model override for Claude Code extraction (e.g. claude-sonnet-4-6).
189    #[arg(long)]
190    pub claude_model: Option<String>,
191
192    /// Resume a previously interrupted claude-code ingest from the queue DB.
193    #[arg(long, default_value_t = false)]
194    pub resume: bool,
195
196    /// Retry only failed files from a previous claude-code ingest.
197    #[arg(long, default_value_t = false)]
198    pub retry_failed: bool,
199
200    /// Keep the queue DB (.ingest-queue.sqlite) after completion.
201    #[arg(long, default_value_t = false)]
202    pub keep_queue: bool,
203
204    /// Custom path for the ingest queue DB. Default: alongside the --db database.
205    #[arg(long)]
206    pub queue_db: Option<String>,
207
208    /// Initial wait time in seconds when rate-limited (only with --mode claude-code).
209    #[arg(long, default_value_t = 60)]
210    pub rate_limit_wait: u64,
211
212    /// Maximum cumulative cost in USD before aborting (only with --mode claude-code).
213    #[arg(long)]
214    pub max_cost_usd: Option<f64>,
215
216    /// Timeout in seconds for each claude -p invocation (only with --mode claude-code).
217    #[arg(
218        long,
219        default_value_t = 300,
220        help = "Timeout in seconds for each claude -p invocation (default: 300)"
221    )]
222    pub claude_timeout: u64,
223
224    /// Explicit path to the Codex CLI binary (only with --mode codex).
225    #[arg(
226        long,
227        help = "Explicit path to the Codex CLI binary (only with --mode codex)"
228    )]
229    pub codex_binary: Option<PathBuf>,
230
231    /// Model override for Codex extraction (e.g. o4-mini, gpt-5.1-codex).
232    #[arg(
233        long,
234        help = "Model override for Codex extraction (e.g. o4-mini, gpt-5.1-codex)"
235    )]
236    pub codex_model: Option<String>,
237
238    /// Timeout in seconds for each codex exec invocation.
239    #[arg(
240        long,
241        default_value_t = 300,
242        help = "Timeout in seconds for each codex exec invocation (default: 300)"
243    )]
244    pub codex_timeout: u64,
245
246    /// Path to the `opencode` binary (override PATH lookup, only with --mode opencode).
247    #[arg(long, value_name = "PATH")]
248    pub opencode_binary: Option<PathBuf>,
249
250    /// Model override for OpenCode extraction.
251    #[arg(
252        long,
253        value_name = "MODEL",
254        help = "Model override for OpenCode extraction"
255    )]
256    pub opencode_model: Option<String>,
257
258    /// Timeout in seconds for each opencode run invocation.
259    #[arg(
260        long,
261        value_name = "SECONDS",
262        default_value_t = 300,
263        help = "Timeout in seconds for each opencode run invocation (default: 300)"
264    )]
265    pub opencode_timeout: u64,
266
267    /// G30: poll for the job singleton every second for up to N seconds
268    /// when another invocation holds the lock. Default: 0 (fail fast).
269    #[arg(long, value_name = "SECONDS")]
270    pub wait_job_singleton: Option<u64>,
271
272    /// G30: force acquisition of the singleton lock by removing a stale
273    /// lock file from a previously crashed invocation.
274    #[arg(long, default_value_t = false)]
275    pub force_job_singleton: bool,
276
277    /// v1.0.93 (GAP-OR-INGEST): run `enrich --operation memory-bindings`
278    /// after all files are embedded, using the active `--llm-backend`.
279    #[arg(
280        long,
281        default_value_t = false,
282        help = "Run enrich --operation memory-bindings after all files are ingested"
283    )]
284    pub enrich_after: bool,
285
286    /// GAP-SG-54: update existing memories instead of skipping them. Without
287    /// this flag a file whose derived name already exists is reported `skipped`;
288    /// with it the existing memory's body, embedding and chunks are refreshed
289    /// (the `remember --force-merge` update path applied per file).
290    #[arg(
291        long,
292        default_value_t = false,
293        help = "Update existing memories on name collision instead of skipping (idempotent re-ingest)"
294    )]
295    pub force_merge: bool,
296}
297
298/// Extraction mode for the ingest pipeline.
299#[derive(Clone, Debug, PartialEq, Eq, clap::ValueEnum)]
300pub enum IngestMode {
301    /// Body-only ingestion without entity/relationship extraction (default).
302    None,
303    /// LLM-curated extraction via locally installed Claude Code CLI.
304    ClaudeCode,
305    /// LLM-curated extraction via locally installed OpenAI Codex CLI.
306    Codex,
307    /// LLM-curated extraction via locally installed OpenCode CLI.
308    #[value(name = "opencode")]
309    Opencode,
310}
311
312/// Returns true when the XDG setting `ingest.low_memory` holds a truthy value
313/// (`1`, `true`, `yes`, `on`, case-insensitive). Empty or unset values evaluate
314/// to false. Unrecognized non-empty values emit a `tracing::warn!` and evaluate
315/// to false.
316///
317/// GAP-SG-83: this function was named `env_low_memory_enabled` and documented an
318/// env var long after `G-T-XDG-04` moved the value to the XDG config, so the log
319/// pointed operators at a variable no reader consults. No product env is read.
320pub(crate) fn low_memory_setting_enabled() -> bool {
321    match crate::config::get_setting("ingest.low_memory") {
322        Ok(Some(v)) if v.is_empty() => false,
323        Ok(Some(v)) => match v.to_lowercase().as_str() {
324            "1" | "true" | "yes" | "on" => true,
325            "0" | "false" | "no" | "off" => false,
326            other => {
327                tracing::warn!(
328                    target: "ingest",
329                    value = %other,
330                    "ingest.low_memory value not recognized; treating as disabled"
331                );
332                false
333            }
334        },
335        _ => false,
336    }
337}
338
339/// Resolves the effective ingest parallelism honoring `--low-memory` and the
340/// XDG setting `ingest.low_memory`.
341///
342/// Precedence (G-T-XDG-04):
343/// 1. `--low-memory` CLI flag forces parallelism = 1.
344/// 2. XDG `ingest.low_memory` truthy forces parallelism = 1.
345/// 3. Explicit `--ingest-parallelism N` (when low-memory is off).
346/// 4. Default heuristic `(cpus/2).clamp(1, 4)`.
347///
348/// When low-memory wins and the user also passed `--ingest-parallelism N>1`,
349/// emits a `tracing::warn!` advertising the override.
350pub(crate) fn resolve_parallelism(
351    low_memory_flag: bool,
352    ingest_parallelism: Option<usize>,
353) -> usize {
354    let setting_flag = low_memory_setting_enabled();
355    let low_memory = low_memory_flag || setting_flag;
356
357    if low_memory {
358        if let Some(n) = ingest_parallelism {
359            if n > 1 {
360                tracing::warn!(
361                    target: "ingest",
362                    requested = n,
363                    "--ingest-parallelism overridden by --low-memory; using 1"
364                );
365            }
366        }
367        if low_memory_flag {
368            tracing::info!(
369                target: "ingest",
370                source = "flag",
371                "low-memory mode enabled: forcing --ingest-parallelism 1"
372            );
373        } else {
374            tracing::info!(
375                target: "ingest",
376                source = "xdg",
377                "low-memory mode enabled via XDG ingest.low_memory: forcing --ingest-parallelism 1"
378            );
379        }
380        return 1;
381    }
382
383    ingest_parallelism
384        .unwrap_or_else(|| {
385            std::thread::available_parallelism()
386                .map(|v| v.get() / 2)
387                .unwrap_or(1)
388                .clamp(1, 4)
389        })
390        .max(1)
391}
sqlite_graphrag/commands/ingest/args.rs

sqlite_graphrag/commands/ingest/
args.rs
