sqlite_graphrag/commands/ingest/
args.rs

1//! CLI arguments and low-memory / parallelism resolution for `ingest`.
2
3use crate::cli::MemoryType;
4use crate::output::JsonOutputFormat;
5use std::path::PathBuf;
6
7#[derive(clap::Args)]
8#[command(after_long_help = "EXAMPLES:\n  \
9    # Ingest every Markdown file under ./docs as `document` memories\n  \
10    sqlite-graphrag ingest ./docs --type document\n\n  \
11    # Ingest .txt files recursively under ./notes\n  \
12    sqlite-graphrag ingest ./notes --type note --pattern '*.txt' --recursive\n\n  \
13    # Namespace derived names with a kebab-case prefix (projx-<derived>)\n  \
14    sqlite-graphrag ingest ./docs --name-prefix projx- --dry-run\n\n  \
15    # Enable automatic URL extraction (URL-regex only since v1.0.79)\n  \
16    sqlite-graphrag ingest ./big-corpus --type reference --enable-ner\n\n  \
17    # Preview file-to-name mapping without ingesting\n  \
18    sqlite-graphrag ingest ./docs --dry-run\n\n  \
19NOTES:\n  \
20    Each file becomes a separate memory. Names derive from file basenames\n  \
21    (kebab-case, lowercase, ASCII). Output is NDJSON: one JSON object per file,\n  \
22    followed by a final summary line with counts. Per-file errors are reported\n  \
23    inline and processing continues unless --fail-fast is set.")]
24/// Ingest args.
25pub struct IngestArgs {
26    /// Directory containing files to ingest.
27    #[arg(
28        value_name = "DIR",
29        help = "Directory to ingest recursively (each matching file becomes a memory)"
30    )]
31    pub dir: PathBuf,
32
33    /// Memory type stored in `memories.type` for every ingested file. Defaults to `document`.
34    #[arg(long, value_enum, default_value_t = MemoryType::Document)]
35    pub r#type: MemoryType,
36
37    /// Glob pattern matched against file basenames (default: `*.md`). Supports
38    /// `*.<ext>`, `<prefix>*`, and exact filename match.
39    #[arg(long, default_value = "*.md")]
40    pub pattern: String,
41
42    /// Recurse into subdirectories.
43    #[arg(long, default_value_t = false)]
44    pub recursive: bool,
45
46    #[arg(
47        long,
48                value_parser = crate::parsers::parse_bool_flexible,
49        action = clap::ArgAction::Set,
50        num_args = 0..=1,
51        default_missing_value = "true",
52        default_value = "false",
53        help = "Enable automatic URL-regex extraction (URL-regex only since v1.0.79)"
54    )]
55    /// Enable NER.
56    pub enable_ner: bool,
57
58    /// GAP-E2E-011: generates a heuristic description from the first meaningful
59    /// line of the body, instead of "ingested from `<path>`". When
60    /// `--no-auto-describe` is passed, keeps the legacy behaviour.
61    #[arg(
62        long,
63        default_value_t = true,
64        overrides_with = "no_auto_describe",
65        help = "Derive memory description from the first meaningful body line instead of the legacy `ingested from <path>` placeholder."
66    )]
67    pub auto_describe: bool,
68    #[arg(
69        long = "no-auto-describe",
70        default_value_t = false,
71        help = "Disable `--auto-describe` and fall back to the legacy `ingested from <path>` description placeholder."
72    )]
73    /// No auto describe.
74    pub no_auto_describe: bool,
75
76    /// Deprecated: NER is now disabled by default. Kept for backwards compatibility.
77    #[arg(long, default_value_t = false, hide = true)]
78    pub skip_extraction: bool,
79
80    /// Stop on first per-file error instead of continuing with the next file.
81    #[arg(long, default_value_t = false)]
82    pub fail_fast: bool,
83
84    /// Preview file-to-name mapping without loading model or persisting.
85    #[arg(long, default_value_t = false)]
86    pub dry_run: bool,
87
88    /// Maximum number of files to ingest (safety cap to prevent runaway ingestion).
89    #[arg(long, default_value_t = 10_000)]
90    pub max_files: usize,
91
92    /// Namespace for the ingested memories.
93    #[arg(long)]
94    pub namespace: Option<String>,
95
96    /// Database path. Falls back to XDG `db.path`, then `graphrag.sqlite`
97    /// under the XDG data directory.
98    #[arg(long)]
99    pub db: Option<String>,
100
101    /// Output format.
102    #[arg(long, value_enum, default_value_t = JsonOutputFormat::Json)]
103    pub format: JsonOutputFormat,
104
105    /// Emit machine-readable JSON on stdout.
106    #[arg(long, hide = true, help = "No-op; JSON is always emitted on stdout")]
107    pub json: bool,
108
109    /// Number of files to extract+embed in parallel; default = max(1, cpus/2).min(4).
110    #[arg(
111        long,
112        help = "Number of files to extract+embed in parallel; default = max(1, cpus/2).min(4)"
113    )]
114    pub ingest_parallelism: Option<usize>,
115
116    /// Force single-threaded ingest to reduce RSS pressure.
117    ///
118    /// Equivalent to `--ingest-parallelism 1`, takes precedence over any
119    /// explicit value. Recommended for environments with <4 GB available
120    /// RAM or container/cgroup constraints. Trade-off: 3-4x longer wall
121    /// time. Also available via XDG `config set ingest.low_memory 1`, which the
122    /// flag overrides. No environment variable supplies this value.
123    #[arg(
124        long,
125        default_value_t = false,
126        help = "Forces single-threaded ingest (--ingest-parallelism 1) to reduce RSS pressure. \
127                Recommended for environments with <4 GB available RAM or container/cgroup \
128                constraints. Trade-off: 3-4x longer wall time. Also honored via \
129                XDG ingest.low_memory=1."
130    )]
131    pub low_memory: bool,
132
133    /// Maximum process RSS in MiB; abort if exceeded during embedding.
134    #[arg(long, default_value_t = crate::constants::DEFAULT_MAX_RSS_MB,
135          help = "Maximum process RSS in MiB; abort if exceeded during embedding (default: 8192)")]
136    pub max_rss_mb: u64,
137
138    /// G42/S3 (v1.0.79): maximum simultaneous LLM embedding subprocesses
139    /// PER FILE. Multiplies with --ingest-parallelism (files staged
140    /// concurrently), hence the conservative default of 2. The effective
141    /// value is further bounded by CPU count and available RAM.
142    #[arg(long, default_value_t = 2, value_name = "N",
143          value_parser = clap::value_parser!(u64).range(1..=32),
144          help = "Maximum simultaneous LLM embedding subprocesses per file (default: 2, clamp [1,32])")]
145    pub llm_parallelism: u64,
146
147    /// Maximum character length for derived memory names from file basenames.
148    ///
149    /// Overrides the compile-time `DERIVED_NAME_MAX_LEN` constant (default 60).
150    /// Shorter values leave more headroom for collision suffix resolution.
151    #[arg(long, default_value_t = crate::constants::DERIVED_NAME_MAX_LEN,
152          help = "Maximum length for derived memory names (default: 60)")]
153    pub max_name_length: usize,
154
155    /// v1.1.1 (P12): kebab-case prefix prepended to every derived memory name,
156    /// AFTER the basename is normalized. Namespaces a corpus inside a shared
157    /// database (e.g. `--name-prefix projx-` yields `projx-<derived>`). The
158    /// derived part's budget shrinks so the final name always respects the
159    /// 80-char name cap. Only supported with `--mode none`.
160    #[arg(
161        long,
162        value_name = "PREFIX",
163        help = "Kebab-case prefix applied to every derived memory name (e.g. 'projx-')"
164    )]
165    pub name_prefix: Option<String>,
166
167    /// Extraction mode: `none` (body-only, default).
168    #[arg(long, value_enum, default_value_t = IngestMode::None)]
169    pub mode: IngestMode,
170
171    /// Maximum cumulative cost in USD before aborting.
172    #[arg(long)]
173    pub max_cost_usd: Option<f64>,
174
175    /// G30: poll for the job singleton every second for up to N seconds
176    /// when another invocation holds the lock. Default: 0 (fail fast).
177    #[arg(long, value_name = "SECONDS")]
178    pub wait_job_singleton: Option<u64>,
179
180    /// G30: force acquisition of the singleton lock by removing a stale
181    /// lock file from a previously crashed invocation.
182    #[arg(long, default_value_t = false)]
183    pub force_job_singleton: bool,
184
185    /// v1.0.93 (GAP-OR-INGEST): run `enrich --operation memory-bindings`
186    /// after all files are embedded, using the active `--llm-backend`.
187    #[arg(
188        long,
189        default_value_t = false,
190        help = "Run enrich --operation memory-bindings after all files are ingested"
191    )]
192    pub enrich_after: bool,
193
194    /// GAP-SG-54: update existing memories instead of skipping them. Without
195    /// this flag a file whose derived name already exists is reported `skipped`;
196    /// with it the existing memory's body, embedding and chunks are refreshed
197    /// (the `remember --force-merge` update path applied per file).
198    #[arg(
199        long,
200        default_value_t = false,
201        help = "Update existing memories on name collision instead of skipping (idempotent re-ingest)"
202    )]
203    pub force_merge: bool,
204}
205
206/// Extraction mode for the ingest pipeline.
207#[derive(Clone, Debug, PartialEq, Eq, clap::ValueEnum)]
208pub enum IngestMode {
209    /// Body-only ingestion without entity/relationship extraction (default).
210    None,
211}
212
213/// Returns true when the XDG setting `ingest.low_memory` holds a truthy value
214/// (`1`, `true`, `yes`, `on`, case-insensitive). Empty or unset values evaluate
215/// to false. Unrecognized non-empty values emit a `tracing::warn!` and evaluate
216/// to false.
217///
218/// GAP-SG-83: this function was named `env_low_memory_enabled` and documented an
219/// env var long after `G-T-XDG-04` moved the value to the XDG config, so the log
220/// pointed operators at a variable no reader consults. No product env is read.
221pub(crate) fn low_memory_setting_enabled() -> bool {
222    match crate::config::get_setting("ingest.low_memory") {
223        Ok(Some(v)) if v.is_empty() => false,
224        Ok(Some(v)) => match v.to_lowercase().as_str() {
225            "1" | "true" | "yes" | "on" => true,
226            "0" | "false" | "no" | "off" => false,
227            other => {
228                tracing::warn!(
229                    target: "ingest",
230                    value = %other,
231                    "ingest.low_memory value not recognized; treating as disabled"
232                );
233                false
234            }
235        },
236        _ => false,
237    }
238}
239
240/// Resolves the effective ingest parallelism honoring `--low-memory` and the
241/// XDG setting `ingest.low_memory`.
242///
243/// Precedence (G-T-XDG-04):
244/// 1. `--low-memory` CLI flag forces parallelism = 1.
245/// 2. XDG `ingest.low_memory` truthy forces parallelism = 1.
246/// 3. Explicit `--ingest-parallelism N` (when low-memory is off).
247/// 4. Default heuristic `(cpus/2).clamp(1, 4)`.
248///
249/// When low-memory wins and the user also passed `--ingest-parallelism N>1`,
250/// emits a `tracing::warn!` advertising the override.
251pub(crate) fn resolve_parallelism(
252    low_memory_flag: bool,
253    ingest_parallelism: Option<usize>,
254) -> usize {
255    let setting_flag = low_memory_setting_enabled();
256    let low_memory = low_memory_flag || setting_flag;
257
258    if low_memory {
259        if let Some(n) = ingest_parallelism {
260            if n > 1 {
261                tracing::warn!(
262                    target: "ingest",
263                    requested = n,
264                    "--ingest-parallelism overridden by --low-memory; using 1"
265                );
266            }
267        }
268        if low_memory_flag {
269            tracing::info!(
270                target: "ingest",
271                source = "flag",
272                "low-memory mode enabled: forcing --ingest-parallelism 1"
273            );
274        } else {
275            tracing::info!(
276                target: "ingest",
277                source = "xdg",
278                "low-memory mode enabled via XDG ingest.low_memory: forcing --ingest-parallelism 1"
279            );
280        }
281        return 1;
282    }
283
284    ingest_parallelism
285        .unwrap_or_else(|| {
286            std::thread::available_parallelism()
287                .map(|v| v.get() / 2)
288                .unwrap_or(1)
289                .clamp(1, 4)
290        })
291        .max(1)
292}
sqlite_graphrag/commands/ingest/args.rs

sqlite_graphrag/commands/ingest/
args.rs
