offline_intelligence/memory_db/
local_files_store.rs

1//! Local files store - manages persistent file metadata in database
2//!
3//! Files are stored with metadata in SQLite and actual content in the app data folder.
4//! Supports nested folder hierarchy.
5
6use std::path::{Path, PathBuf};
7use std::sync::Arc;
8use r2d2::Pool;
9use r2d2_sqlite::SqliteConnectionManager;
10use serde::{Deserialize, Serialize};
11use chrono::{DateTime, Utc};
12use tracing::info;
13
14/// Represents a local file or directory
15#[derive(Debug, Clone, Serialize, Deserialize)]
16pub struct LocalFile {
17    pub id: i64,
18    pub name: String,
19    pub path: String,
20    pub parent_id: Option<i64>,
21    pub is_directory: bool,
22    pub file_path: Option<String>,
23    pub size_bytes: i64,
24    pub mime_type: Option<String>,
25    pub created_at: DateTime<Utc>,
26    pub modified_at: DateTime<Utc>,
27}
28
29/// Represents a file tree node with children (for nested display)
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct LocalFileTree {
32    #[serde(flatten)]
33    pub file: LocalFile,
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub children: Option<Vec<LocalFileTree>>,
36}
37
38/// Store for managing local files in database
39pub struct LocalFilesStore {
40    pool: Arc<Pool<SqliteConnectionManager>>,
41    app_data_dir: PathBuf,
42}
43
44impl LocalFilesStore {
45    /// Create a new local files store
46    pub fn new(pool: Arc<Pool<SqliteConnectionManager>>, app_data_dir: PathBuf) -> Self {
47        Self { pool, app_data_dir }
48    }
49
50    /// Get the app data directory path
51    pub fn get_app_data_dir(&self) -> &Path {
52        &self.app_data_dir
53    }
54
55    /// Create a folder
56    pub fn create_folder(&self, parent_id: Option<i64>, name: &str) -> anyhow::Result<LocalFile> {
57        let conn = self.pool.get()?;
58        
59        // Build the path
60        let path = if let Some(pid) = parent_id {
61            let parent_path: String = conn.query_row(
62                "SELECT path FROM local_files WHERE id = ?1",
63                [pid],
64                |row| row.get(0),
65            )?;
66            format!("{}/{}", parent_path, name)
67        } else {
68            name.to_string()
69        };
70
71        let now = chrono::Utc::now().to_rfc3339();
72        
73        conn.execute(
74            "INSERT INTO local_files (name, path, parent_id, is_directory, created_at, modified_at)
75             VALUES (?1, ?2, ?3, TRUE, ?4, ?4)",
76            rusqlite::params![name, path, parent_id, now],
77        )?;
78
79        let id = conn.last_insert_rowid();
80        
81        // Create actual directory in filesystem
82        let fs_path = self.app_data_dir.join(&path);
83        std::fs::create_dir_all(&fs_path)?;
84        
85        info!("Created folder: {} (id: {})", path, id);
86        
87        self.get_file(id)
88    }
89
90    /// Upload a file
91    pub fn upload_file(
92        &self,
93        parent_id: Option<i64>,
94        name: &str,
95        content: &[u8],
96        mime_type: Option<&str>,
97    ) -> anyhow::Result<LocalFile> {
98        let conn = self.pool.get()?;
99        
100        // Build the path
101        let path = if let Some(pid) = parent_id {
102            let parent_path: String = conn.query_row(
103                "SELECT path FROM local_files WHERE id = ?1",
104                [pid],
105                |row| row.get(0),
106            )?;
107            format!("{}/{}", parent_path, name)
108        } else {
109            name.to_string()
110        };
111
112        // Save file to filesystem
113        let fs_path = self.app_data_dir.join(&path);
114        if let Some(parent) = fs_path.parent() {
115            std::fs::create_dir_all(parent)?;
116        }
117        std::fs::write(&fs_path, content)?;
118
119        let now = chrono::Utc::now().to_rfc3339();
120        let size = content.len() as i64;
121        
122        // Check if file already exists (update) or create new
123        let existing_id: Option<i64> = conn.query_row(
124            "SELECT id FROM local_files WHERE parent_id IS ?1 AND name = ?2",
125            rusqlite::params![parent_id, name],
126            |row| row.get(0),
127        ).ok();
128
129        let id = if let Some(existing) = existing_id {
130            // Update existing file
131            conn.execute(
132                "UPDATE local_files SET file_path = ?1, size_bytes = ?2, mime_type = ?3, modified_at = ?4
133                 WHERE id = ?5",
134                rusqlite::params![path, size, mime_type, now, existing],
135            )?;
136            existing
137        } else {
138            // Insert new file
139            conn.execute(
140                "INSERT INTO local_files (name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at)
141                 VALUES (?1, ?2, ?3, FALSE, ?2, ?4, ?5, ?6, ?6)",
142                rusqlite::params![name, path, parent_id, size, mime_type, now],
143            )?;
144            conn.last_insert_rowid()
145        };
146        
147        info!("Uploaded file: {} ({} bytes, id: {})", path, size, id);
148        
149        self.get_file(id)
150    }
151
152    /// List files in a directory (or root if parent_id is None)
153    pub fn list_files(&self, parent_id: Option<i64>) -> anyhow::Result<Vec<LocalFile>> {
154        let conn = self.pool.get()?;
155        
156        let files: Vec<LocalFile> = if let Some(pid) = parent_id {
157            let mut stmt = conn.prepare(
158                "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
159                 FROM local_files WHERE parent_id = ?1
160                 ORDER BY is_directory DESC, name ASC"
161            )?;
162            let rows = stmt.query_map([pid], |row| self.row_to_local_file(row))?;
163            rows.filter_map(|r| r.ok()).collect()
164        } else {
165            let mut stmt = conn.prepare(
166                "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
167                 FROM local_files WHERE parent_id IS NULL
168                 ORDER BY is_directory DESC, name ASC"
169            )?;
170            let rows = stmt.query_map([], |row| self.row_to_local_file(row))?;
171            rows.filter_map(|r| r.ok()).collect()
172        };
173
174        Ok(files)
175    }
176
177    /// Get a single file by ID
178    pub fn get_file(&self, id: i64) -> anyhow::Result<LocalFile> {
179        let conn = self.pool.get()?;
180        
181        let file = conn.query_row(
182            "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
183             FROM local_files WHERE id = ?1",
184            [id],
185            |row| self.row_to_local_file(row),
186        )?;
187        
188        Ok(file)
189    }
190
191    /// Get file by path
192    pub fn get_file_by_path(&self, path: &str) -> anyhow::Result<LocalFile> {
193        let conn = self.pool.get()?;
194        
195        let file = conn.query_row(
196            "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
197             FROM local_files WHERE path = ?1",
198            [path],
199            |row| self.row_to_local_file(row),
200        )?;
201        
202        Ok(file)
203    }
204
205    /// Get file by name (searches all files, returns first match)
206    pub fn get_file_by_name(&self, name: &str) -> anyhow::Result<LocalFile> {
207        let conn = self.pool.get()?;
208        
209        let file = conn.query_row(
210            "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
211             FROM local_files WHERE name = ?1 AND is_directory = FALSE
212             ORDER BY modified_at DESC LIMIT 1",
213            [name],
214            |row| self.row_to_local_file(row),
215        )?;
216        
217        Ok(file)
218    }
219
220    /// Check if an entry (file or folder) exists with given name and parent
221    fn entry_exists(&self, parent_id: Option<i64>, name: &str) -> anyhow::Result<Option<LocalFile>> {
222        let conn = self.pool.get()?;
223        
224        let result = if let Some(pid) = parent_id {
225            conn.query_row(
226                "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
227                 FROM local_files WHERE parent_id = ?1 AND name = ?2",
228                rusqlite::params![pid, name],
229                |row| self.row_to_local_file(row),
230            )
231        } else {
232            conn.query_row(
233                "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
234                 FROM local_files WHERE parent_id IS NULL AND name = ?1",
235                [name],
236                |row| self.row_to_local_file(row),
237            )
238        };
239        
240        match result {
241            Ok(file) => Ok(Some(file)),
242            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
243            Err(e) => Err(e.into()),
244        }
245    }
246
247    /// Get file content by ID
248    pub fn get_file_content(&self, id: i64) -> anyhow::Result<Vec<u8>> {
249        let file = self.get_file(id)?;
250        
251        if file.is_directory {
252            anyhow::bail!("Cannot read content of a directory");
253        }
254        
255        let file_path = file.file_path.ok_or_else(|| anyhow::anyhow!("File has no path"))?;
256        let fs_path = self.app_data_dir.join(&file_path);
257        
258        let content = std::fs::read(&fs_path)?;
259        Ok(content)
260    }
261
262    /// Get file content as string
263    pub fn get_file_content_string(&self, id: i64) -> anyhow::Result<String> {
264        let content = self.get_file_content(id)?;
265        let text = String::from_utf8(content)?;
266        Ok(text)
267    }
268
269    /// Delete a file or folder (recursively)
270    pub fn delete_file(&self, id: i64) -> anyhow::Result<()> {
271        let file = self.get_file(id)?;
272        let conn = self.pool.get()?;
273        
274        // Delete from filesystem
275        let fs_path = self.app_data_dir.join(&file.path);
276        if file.is_directory {
277            if fs_path.exists() {
278                std::fs::remove_dir_all(&fs_path)?;
279            }
280        } else if fs_path.exists() {
281            std::fs::remove_file(&fs_path)?;
282        }
283        
284        // Delete from database (CASCADE will handle children)
285        conn.execute("DELETE FROM local_files WHERE id = ?1", [id])?;
286        
287        info!("Deleted: {} (id: {})", file.path, id);
288        
289        Ok(())
290    }
291
292    /// Search files by name (partial match)
293    pub fn search_files(&self, query: &str) -> anyhow::Result<Vec<LocalFile>> {
294        let conn = self.pool.get()?;
295        let pattern = format!("%{}%", query);
296        
297        let mut stmt = conn.prepare(
298            "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
299             FROM local_files WHERE name LIKE ?1
300             ORDER BY is_directory DESC, name ASC
301             LIMIT 50"
302        )?;
303
304        let rows = stmt.query_map([pattern], |row| self.row_to_local_file(row))?;
305        let files: Vec<LocalFile> = rows.filter_map(|r| r.ok()).collect();
306        
307        Ok(files)
308    }
309
310    /// Get the complete file tree (nested structure)
311    pub fn get_file_tree(&self) -> anyhow::Result<Vec<LocalFileTree>> {
312        let root_files = self.list_files(None)?;
313        let tree = self.build_tree(root_files)?;
314        Ok(tree)
315    }
316
317    /// Build nested tree recursively
318    fn build_tree(&self, files: Vec<LocalFile>) -> anyhow::Result<Vec<LocalFileTree>> {
319        let mut tree = Vec::new();
320        
321        for file in files {
322            let children = if file.is_directory {
323                let child_files = self.list_files(Some(file.id))?;
324                if child_files.is_empty() {
325                    None
326                } else {
327                    Some(self.build_tree(child_files)?)
328                }
329            } else {
330                None
331            };
332            
333            tree.push(LocalFileTree { file, children });
334        }
335        
336        Ok(tree)
337    }
338
339    /// Get all files (flat list)
340    pub fn get_all_files(&self) -> anyhow::Result<Vec<LocalFile>> {
341        let conn = self.pool.get()?;
342        
343        let mut stmt = conn.prepare(
344            "SELECT id, name, path, parent_id, is_directory, file_path, size_bytes, mime_type, created_at, modified_at
345             FROM local_files WHERE is_directory = FALSE
346             ORDER BY name ASC"
347        )?;
348
349        let rows = stmt.query_map([], |row| self.row_to_local_file(row))?;
350        let files: Vec<LocalFile> = rows.filter_map(|r| r.ok()).collect();
351        
352        Ok(files)
353    }
354
355    /// Helper to convert a row to LocalFile
356    fn row_to_local_file(&self, row: &rusqlite::Row<'_>) -> rusqlite::Result<LocalFile> {
357        let created_at_str: String = row.get(8)?;
358        let modified_at_str: String = row.get(9)?;
359        
360        let created_at = chrono::DateTime::parse_from_rfc3339(&created_at_str)
361            .map(|dt| dt.with_timezone(&chrono::Utc))
362            .unwrap_or_else(|_| chrono::Utc::now());
363        
364        let modified_at = chrono::DateTime::parse_from_rfc3339(&modified_at_str)
365            .map(|dt| dt.with_timezone(&chrono::Utc))
366            .unwrap_or_else(|_| chrono::Utc::now());
367        
368        Ok(LocalFile {
369            id: row.get(0)?,
370            name: row.get(1)?,
371            path: row.get(2)?,
372            parent_id: row.get(3)?,
373            is_directory: row.get(4)?,
374            file_path: row.get(5)?,
375            size_bytes: row.get(6)?,
376            mime_type: row.get(7)?,
377            created_at,
378            modified_at,
379        })
380    }
381
382    /// Sync filesystem with database (import existing files)
383    pub fn sync_from_filesystem(&self) -> anyhow::Result<usize> {
384        let mut imported = 0;
385        
386        if !self.app_data_dir.exists() {
387            std::fs::create_dir_all(&self.app_data_dir)?;
388            return Ok(0);
389        }
390        
391        imported += self.import_directory(None, &self.app_data_dir)?;
392        
393        info!("Synced {} files/folders from filesystem", imported);
394        Ok(imported)
395    }
396
397    /// Clear all entries from local_files table (for cleanup/reset)
398    pub fn clear_all(&self) -> anyhow::Result<usize> {
399        let conn = self.pool.get()?;
400        let deleted = conn.execute("DELETE FROM local_files", [])?;
401        info!("Cleared {} entries from local_files", deleted);
402        Ok(deleted)
403    }
404
405    /// Import a directory recursively
406    fn import_directory(&self, parent_id: Option<i64>, dir_path: &Path) -> anyhow::Result<usize> {
407        let mut count = 0;
408        
409        // System directories to exclude from local files browser
410        const EXCLUDED_DIRS: &[&str] = &["models", "registry"];
411        
412        for entry in std::fs::read_dir(dir_path)? {
413            let entry = entry?;
414            let name = entry.file_name().to_string_lossy().to_string();
415            let metadata = entry.metadata()?;
416            
417            // Skip system files and excluded directories
418            if name.starts_with('.') || name == "conversations.db" || name.ends_with("-wal") || name.ends_with("-shm") {
419                continue;
420            }
421            
422            // Skip system directories at root level
423            if parent_id.is_none() && EXCLUDED_DIRS.contains(&name.as_str()) {
424                continue;
425            }
426            
427            if metadata.is_dir() {
428                // Check if folder exists in DB (by parent_id and name)
429                let folder_id = if let Some(existing) = self.entry_exists(parent_id, &name)? {
430                    existing.id
431                } else {
432                    let folder = self.create_folder(parent_id, &name)?;
433                    count += 1;
434                    folder.id
435                };
436                
437                // Recurse into subdirectory
438                count += self.import_directory(Some(folder_id), &entry.path())?;
439            } else {
440                // Check if file exists in DB (by parent_id and name)
441                if self.entry_exists(parent_id, &name)?.is_none() {
442                    let content = std::fs::read(entry.path())?;
443                    let mime = mime_guess::from_path(&entry.path())
444                        .first()
445                        .map(|m| m.to_string());
446                    
447                    self.upload_file(parent_id, &name, &content, mime.as_deref())?;
448                    count += 1;
449                }
450            }
451        }
452        
453        Ok(count)
454    }
455}
456
offline_intelligence/memory_db/local_files_store.rs

offline_intelligence/memory_db/
local_files_store.rs
