systemprompt_content/config/
ready.rs

1//! Loaded content index built from a validated configuration.
2//!
3//! [`ContentReady`] scans every enabled source's directory, parses each
4//! markdown file's frontmatter into [`ParsedContent`], and indexes the results
5//! by slug and by [`SourceId`]. [`LoadStats`] records per-source scan and parse
6//! outcomes for diagnostics.
7//!
8//! Copyright (c) systemprompt.io — Business Source License 1.1.
9//! See <https://systemprompt.io> for licensing details.
10
11use chrono::{DateTime, Utc};
12use sha2::{Digest, Sha256};
13use std::collections::HashMap;
14use std::path::{Path, PathBuf};
15use systemprompt_identifiers::{CategoryId, SourceId};
16use systemprompt_models::{ContentRouting, split_frontmatter};
17use walkdir::WalkDir;
18
19use crate::ContentError;
20use crate::models::ContentMetadata;
21use crate::services::validate_content_metadata;
22
23use super::validated::{ContentConfigValidated, ContentSourceConfigValidated};
24
25#[derive(Debug, Clone)]
26pub struct ContentReady {
27    config: ContentConfigValidated,
28    content_by_slug: HashMap<String, ParsedContent>,
29    content_by_source: HashMap<SourceId, Vec<ParsedContent>>,
30    stats: LoadStats,
31}
32
33#[derive(Debug, Clone)]
34pub struct ParsedContent {
35    pub slug: String,
36    pub title: String,
37    pub description: String,
38    pub body: String,
39    pub author: String,
40    pub published_at: DateTime<Utc>,
41    pub keywords: String,
42    pub kind: String,
43    pub image: Option<String>,
44    pub category_id: CategoryId,
45    pub source_id: SourceId,
46    pub version_hash: String,
47    pub file_path: PathBuf,
48}
49
50#[derive(Debug, Clone, Default)]
51pub struct LoadStats {
52    pub files_found: usize,
53    pub files_loaded: usize,
54    pub files_with_errors: usize,
55    pub load_time_ms: u64,
56    pub source_stats: HashMap<String, SourceLoadStats>,
57}
58
59#[derive(Debug, Clone, Copy, Default)]
60pub struct SourceLoadStats {
61    pub files_found: usize,
62    pub files_loaded: usize,
63    pub errors: usize,
64}
65
66impl ContentReady {
67    pub fn from_validated(config: ContentConfigValidated) -> Self {
68        let start_time = std::time::Instant::now();
69        let mut content_by_slug = HashMap::new();
70        let mut content_by_source: HashMap<SourceId, Vec<ParsedContent>> = HashMap::new();
71        let mut stats = LoadStats::default();
72
73        for (source_name, source_config) in config.content_sources() {
74            if !source_config.enabled {
75                continue;
76            }
77
78            let mut source_stats = SourceLoadStats::default();
79
80            let files = scan_markdown_files(&source_config.path, source_config.indexing.recursive);
81            source_stats.files_found = files.len();
82            stats.files_found += files.len();
83
84            for file_path in files {
85                match parse_content_file(&file_path, source_config) {
86                    Ok(content) => {
87                        let slug = content.slug.clone();
88                        let source_id = content.source_id.clone();
89
90                        content_by_source
91                            .entry(source_id)
92                            .or_default()
93                            .push(content.clone());
94
95                        content_by_slug.insert(slug, content);
96
97                        source_stats.files_loaded += 1;
98                        stats.files_loaded += 1;
99                    },
100                    Err(e) => {
101                        tracing::warn!(
102                            file = %file_path.display(),
103                            error = %e,
104                            "Failed to parse content file"
105                        );
106                        source_stats.errors += 1;
107                        stats.files_with_errors += 1;
108                    },
109                }
110            }
111
112            stats.source_stats.insert(source_name.clone(), source_stats);
113        }
114
115        stats.load_time_ms = start_time.elapsed().as_millis() as u64;
116
117        Self {
118            config,
119            content_by_slug,
120            content_by_source,
121            stats,
122        }
123    }
124
125    pub const fn config(&self) -> &ContentConfigValidated {
126        &self.config
127    }
128
129    pub const fn stats(&self) -> &LoadStats {
130        &self.stats
131    }
132
133    pub fn get_by_slug(&self, slug: &str) -> Option<&ParsedContent> {
134        self.content_by_slug.get(slug)
135    }
136
137    pub fn get_by_source(&self, source_id: &SourceId) -> Option<&Vec<ParsedContent>> {
138        self.content_by_source.get(source_id)
139    }
140
141    pub fn all_content(&self) -> impl Iterator<Item = &ParsedContent> {
142        self.content_by_slug.values()
143    }
144
145    pub fn content_count(&self) -> usize {
146        self.content_by_slug.len()
147    }
148}
149
150impl ContentRouting for ContentReady {
151    fn is_html_page(&self, path: &str) -> bool {
152        self.config.is_html_page(path)
153    }
154
155    fn determine_source(&self, path: &str) -> String {
156        self.config.determine_source(path)
157    }
158}
159
160fn scan_markdown_files(dir: &Path, recursive: bool) -> Vec<PathBuf> {
161    let walker = if recursive {
162        WalkDir::new(dir).min_depth(1)
163    } else {
164        WalkDir::new(dir).min_depth(1).max_depth(1)
165    };
166
167    walker
168        .into_iter()
169        .filter_map(Result::ok)
170        .filter(|e| e.file_type().is_file())
171        .filter(|e| e.path().extension().is_some_and(|ext| ext == "md"))
172        .map(|e| e.path().to_path_buf())
173        .collect()
174}
175
176fn parse_content_file(
177    file_path: &Path,
178    source_config: &ContentSourceConfigValidated,
179) -> Result<ParsedContent, ContentError> {
180    let markdown_text = std::fs::read_to_string(file_path).map_err(ContentError::Io)?;
181
182    let (metadata, body) = parse_frontmatter(&markdown_text)?;
183
184    validate_content_metadata(&metadata)?;
185
186    let published_at = parse_date(&metadata.published_at)?;
187
188    let category_id = metadata.category.as_ref().map_or_else(
189        || source_config.category_id.clone(),
190        |c| CategoryId::new(c.clone()),
191    );
192
193    let version_hash = compute_version_hash(&metadata.title, &body, &metadata.description);
194
195    Ok(ParsedContent {
196        slug: metadata.slug,
197        title: metadata.title,
198        description: metadata.description,
199        body,
200        author: metadata.author,
201        published_at,
202        keywords: metadata.keywords,
203        kind: metadata.kind,
204        image: metadata.image,
205        category_id,
206        source_id: source_config.source_id.clone(),
207        version_hash,
208        file_path: file_path.to_path_buf(),
209    })
210}
211
212fn parse_frontmatter(markdown: &str) -> Result<(ContentMetadata, String), ContentError> {
213    let frontmatter = split_frontmatter(markdown).ok_or_else(|| {
214        ContentError::Parse("Invalid frontmatter format - missing '---' delimiters".to_owned())
215    })?;
216
217    let metadata: ContentMetadata =
218        serde_yaml::from_str(frontmatter.yaml).map_err(ContentError::Yaml)?;
219
220    Ok((metadata, frontmatter.body.trim().to_owned()))
221}
222
223fn parse_date(date_str: &str) -> Result<DateTime<Utc>, ContentError> {
224    chrono::NaiveDate::parse_from_str(date_str, "%Y-%m-%d")
225        .map_err(|e| ContentError::Parse(format!("Invalid date '{}': {}", date_str, e)))?
226        .and_hms_opt(0, 0, 0)
227        .ok_or_else(|| ContentError::Parse("Failed to create datetime".to_owned()))?
228        .and_local_timezone(Utc)
229        .single()
230        .ok_or_else(|| ContentError::Parse("Ambiguous timezone conversion".to_owned()))
231}
232
233fn compute_version_hash(title: &str, body: &str, description: &str) -> String {
234    let mut hasher = Sha256::new();
235    hasher.update(title.as_bytes());
236    hasher.update(body.as_bytes());
237    hasher.update(description.as_bytes());
238    hex::encode(hasher.finalize())
239}
systemprompt_content/config/ready.rs

systemprompt_content/config/
ready.rs
