datafold 0.1.55

A personal database for data sovereignty with AI-powered ingestion
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
//! Message signing and verification functionality

use crate::log_feature;
use crate::logging::features::LogFeature;
use crate::{
    constants::SINGLE_PUBLIC_KEY_ID,
    db_operations::DbOperations,
    security::{
        Ed25519PublicKey, KeyUtils, PublicKeyInfo, SecurityError, SecurityResult, SignedMessage,
        VerificationResult,
    },
};
use base64::{engine::general_purpose, Engine as _};
use serde_json::Value;
use std::sync::{Arc, RwLock};

/// Message signer for client-side use
pub struct MessageSigner {
    keypair: crate::security::Ed25519KeyPair,
}

impl MessageSigner {
    /// Create a new message signer with a key pair
    pub fn new(keypair: crate::security::Ed25519KeyPair) -> Self {
        Self { keypair }
    }

    /// Sign a message payload
    pub fn sign_message(&self, payload: Value) -> SecurityResult<SignedMessage> {
        // Serialize the payload to canonical JSON
        let payload_bytes = self.serialize_payload(&payload)?;

        // Create timestamp
        let timestamp = chrono::Utc::now().timestamp();

        // Create message to sign (payload + timestamp + key_id)
        let mut message_to_sign = payload_bytes.clone();
        message_to_sign.extend_from_slice(&timestamp.to_be_bytes());
        message_to_sign.extend_from_slice(SINGLE_PUBLIC_KEY_ID.as_bytes());

        // Sign the message
        let signature = self.keypair.sign(&message_to_sign);
        let signature_base64 = KeyUtils::signature_to_base64(&signature);

        // Base64 encode the original payload for storage
        let payload_base64 = general_purpose::STANDARD.encode(&payload_bytes);

        Ok(SignedMessage::new(
            payload_base64,
            SINGLE_PUBLIC_KEY_ID.to_string(),
            signature_base64,
            timestamp,
        ))
    }

    /// Serialize payload to canonical JSON bytes
    fn serialize_payload(&self, payload: &Value) -> SecurityResult<Vec<u8>> {
        serde_json::to_vec(payload).map_err(|e| SecurityError::SerializationError(e.to_string()))
    }
}

/// Message verifier for server-side use with optional persistence
pub struct MessageVerifier {
    /// The registered public key (in-memory cache)
    public_key: Arc<RwLock<Option<PublicKeyInfo>>>,
    /// Database operations for persistence
    db_ops: Option<Arc<DbOperations>>,
    /// Maximum allowed timestamp drift in seconds
    max_timestamp_drift: i64,
}

impl MessageVerifier {
    /// Create a new message verifier without persistence
    pub fn new(max_timestamp_drift: i64) -> Self {
        Self {
            public_key: Arc::new(RwLock::new(None)),
            db_ops: None,
            max_timestamp_drift,
        }
    }

    /// Create a new message verifier with database persistence
    pub async fn new_with_persistence(
        max_timestamp_drift: i64,
        db_ops: Arc<DbOperations>,
    ) -> SecurityResult<Self> {
        let verifier = Self {
            public_key: Arc::new(RwLock::new(None)),
            db_ops: Some(db_ops),
            max_timestamp_drift,
        };

        // Load persisted key from database
        verifier.load_persisted_key_async().await?;
        Ok(verifier)
    }

    /// Load the persisted public key from database into memory
    async fn load_persisted_key_async(&self) -> SecurityResult<()> {
        if let Some(db_ops) = &self.db_ops {
            match db_ops.get_system_public_key().await {
                Ok(Some(persisted_key)) => {
                    let mut key_lock = self.public_key.write().map_err(|_| {
                        SecurityError::KeyNotFound("Failed to acquire write lock".to_string())
                    })?;
                    *key_lock = Some(persisted_key);
                    log_feature!(
                        LogFeature::Permissions,
                        info,
                        "Loaded system public key from database"
                    );
                }
                Ok(None) => {
                    log_feature!(
                        LogFeature::Permissions,
                        info,
                        "No system public key found in database."
                    );
                }
                Err(e) => {
                    log_feature!(
                        LogFeature::Permissions,
                        warn,
                        "Failed to load persisted public key: {}",
                        e
                    );
                    // Don't fail initialization - continue without persisted key
                }
            }
        }
        Ok(())
    }

    /// Persist a public key to database
    async fn persist_public_key(&self, key_info: &PublicKeyInfo) -> SecurityResult<()> {
        if let Some(db_ops) = &self.db_ops {
            match db_ops.store_system_public_key(key_info).await {
                Ok(()) => {
                    log_feature!(
                        LogFeature::Permissions,
                        debug,
                        "Persisted system public key"
                    );
                    Ok(())
                }
                Err(e) => {
                    log_feature!(
                        LogFeature::Permissions,
                        error,
                        "Failed to persist system public key: {}",
                        e
                    );
                    // Don't fail the operation - key is still in memory
                    Ok(())
                }
            }
        } else {
            Ok(())
        }
    }

    /// Register the system-wide public key with automatic persistence
    pub async fn register_system_public_key(&self, key_info: PublicKeyInfo) -> SecurityResult<()> {
        let mut key_to_store = key_info;
        key_to_store.id = SINGLE_PUBLIC_KEY_ID.to_string();

        // Store in memory first
        {
            let mut key = self.public_key.write().map_err(|_| {
                SecurityError::KeyNotFound("Failed to acquire write lock".to_string())
            })?;
            *key = Some(key_to_store.clone());
        }

        // Then persist to database
        self.persist_public_key(&key_to_store).await?;

        log_feature!(
            LogFeature::Permissions,
            info,
            "Registered system public key"
        );
        Ok(())
    }

    /// Remove the system public key from both memory and database
    pub async fn remove_system_public_key(&self) -> SecurityResult<()> {
        // Remove from memory
        {
            let mut key = self.public_key.write().map_err(|_| {
                SecurityError::KeyNotFound("Failed to acquire write lock".to_string())
            })?;
            *key = None;
        }

        // Remove from database
        if let Some(db_ops) = &self.db_ops {
            match db_ops.delete_system_public_key().await {
                Ok(_) => log_feature!(
                    LogFeature::Permissions,
                    debug,
                    "Removed system public key from database"
                ),
                Err(e) => log_feature!(
                    LogFeature::Permissions,
                    error,
                    "Failed to remove system public key from database: {}",
                    e
                ),
            }
        }

        Ok(())
    }

    /// Get the system public key info
    pub fn get_system_public_key(&self) -> SecurityResult<Option<PublicKeyInfo>> {
        Ok(self
            .public_key
            .read()
            .map_err(|_| SecurityError::KeyNotFound("Failed to acquire read lock".to_string()))?
            .clone())
    }

    /// List the system public key if it exists.
    pub fn list_public_keys(&self) -> SecurityResult<Vec<PublicKeyInfo>> {
        let key = self
            .public_key
            .read()
            .map_err(|_| SecurityError::KeyNotFound("Failed to acquire read lock".to_string()))?;

        if let Some(k) = &*key {
            Ok(vec![k.clone()])
        } else {
            Ok(vec![])
        }
    }

    /// Verify a signed message
    pub fn verify_message(
        &self,
        signed_message: &SignedMessage,
    ) -> SecurityResult<VerificationResult> {
        // Get the public key info
        let key_info = match self.get_system_public_key()? {
            Some(info) => info,
            None => {
                return Ok(VerificationResult::failure(
                    "System public key not found".to_string(),
                ))
            }
        };

        // Check if key is valid (not expired, active, etc.)
        if !key_info.is_valid() {
            return Ok(VerificationResult::failure(
                "Public key is not valid (expired or inactive)".to_string(),
            ));
        }

        // Check timestamp validity
        let timestamp_valid = self.is_timestamp_valid(signed_message.timestamp);

        // Parse the public key
        let public_key = match Ed25519PublicKey::from_base64(&key_info.public_key) {
            Ok(key) => key,
            Err(e) => {
                return Ok(VerificationResult::failure(format!(
                    "Invalid public key format: {}",
                    e
                )))
            }
        };

        // Parse the signature
        let signature = match KeyUtils::signature_from_base64(&signed_message.signature) {
            Ok(sig) => sig,
            Err(e) => {
                return Ok(VerificationResult::failure(format!(
                    "Invalid signature format: {}",
                    e
                )))
            }
        };

        // Recreate the message that was signed
        let message_to_verify = match self.create_message_to_verify(signed_message) {
            Ok(msg) => msg,
            Err(e) => {
                return Ok(VerificationResult::failure(format!(
                    "Failed to recreate message: {}",
                    e
                )))
            }
        };

        // Verify the signature
        let is_valid = public_key.verify(&message_to_verify, &signature);

        if is_valid && timestamp_valid {
            Ok(VerificationResult::success(key_info, timestamp_valid))
        } else {
            Ok(VerificationResult::failure(
                "Signature verification failed".to_string(),
            ))
        }
    }

    /// Check if timestamp is within acceptable range
    fn is_timestamp_valid(&self, timestamp: i64) -> bool {
        let current_time = chrono::Utc::now().timestamp();
        (current_time - timestamp).abs() <= self.max_timestamp_drift
    }

    /// Recreate the original signed message for verification
    fn create_message_to_verify(&self, signed_message: &SignedMessage) -> SecurityResult<Vec<u8>> {
        let mut message = general_purpose::STANDARD
            .decode(&signed_message.payload)
            .map_err(|e| SecurityError::DeserializationError(e.to_string()))?;
        message.extend_from_slice(&signed_message.timestamp.to_be_bytes());
        message.extend_from_slice(signed_message.public_key_id.as_bytes());
        Ok(message)
    }

    /// Check permissions and verify message
    pub fn verify_message_with_permissions(
        &self,
        signed_message: &SignedMessage,
        required_permissions: &[String],
    ) -> SecurityResult<VerificationResult> {
        let verification_result = self.verify_message(signed_message)?;
        if !verification_result.is_valid {
            return Ok(verification_result);
        }

        if let Some(key_info) = &verification_result.public_key_info {
            for perm in required_permissions {
                if !key_info.permissions.contains(perm) {
                    return Ok(VerificationResult::failure(format!(
                        "Missing required permission: {}",
                        perm
                    )));
                }
            }
        }

        Ok(verification_result)
    }
}

/// Utility functions for signing
pub struct SigningUtils;

impl SigningUtils {
    /// Create a signer from a base64 encoded secret key
    pub fn create_signer_from_secret(secret_key_base64: &str) -> SecurityResult<MessageSigner> {
        let secret_key_bytes = general_purpose::STANDARD
            .decode(secret_key_base64)
            .map_err(|e| SecurityError::InvalidKeyFormat(e.to_string()))?;
        let keypair = crate::security::Ed25519KeyPair::from_secret_key(&secret_key_bytes)?;
        Ok(MessageSigner::new(keypair))
    }

    /// Get the owner ID from a verification result
    pub fn get_message_owner(verification_result: &VerificationResult) -> Option<String> {
        verification_result
            .public_key_info
            .as_ref()
            .map(|info| info.owner_id.clone())
    }

    /// Check if verification was successful
    pub fn is_verification_successful(result: &VerificationResult) -> bool {
        result.is_valid
    }
}

#[cfg(test)]
mod persistence_tests {
    use super::*;
    use crate::security::Ed25519KeyPair;
    use crate::testing_utils::TestDatabaseFactory;

    #[tokio::test]
    async fn test_message_verifier_persistence() {
        let (db_ops, _) = TestDatabaseFactory::create_test_environment()
            .await
            .unwrap();
        let verifier = MessageVerifier::new_with_persistence(60, db_ops.clone())
            .await
            .unwrap();
        let keypair = Ed25519KeyPair::generate().unwrap();
        let public_key_base64 = keypair.public_key_base64();

        let key_info = PublicKeyInfo::new(
            "test_key".to_string(),
            public_key_base64,
            "test_owner".to_string(),
            vec!["read".to_string()],
        );
        verifier
            .register_system_public_key(key_info.clone())
            .await
            .unwrap();

        // Now create a new verifier with the same database
        let verifier2 = MessageVerifier::new_with_persistence(60, db_ops)
            .await
            .unwrap();
        let retrieved = verifier2.get_system_public_key().unwrap();
        assert!(retrieved.is_some());
        assert_eq!(retrieved.unwrap().id, SINGLE_PUBLIC_KEY_ID.to_string());
    }

    #[tokio::test]
    async fn test_remove_public_key_persistence() {
        let (db_ops, _) = TestDatabaseFactory::create_test_environment()
            .await
            .unwrap();
        let verifier = MessageVerifier::new_with_persistence(60, db_ops.clone())
            .await
            .unwrap();
        let keypair = Ed25519KeyPair::generate().unwrap();
        let public_key_base64 = keypair.public_key_base64();

        let key_info = PublicKeyInfo::new(
            "test_key".to_string(),
            public_key_base64,
            "test_owner".to_string(),
            vec!["read".to_string()],
        );
        verifier.register_system_public_key(key_info).await.unwrap();
        assert!(verifier.get_system_public_key().unwrap().is_some());

        verifier.remove_system_public_key().await.unwrap();
        assert!(verifier.get_system_public_key().unwrap().is_none());

        // Verify with a new verifier instance
        let verifier2 = MessageVerifier::new_with_persistence(60, db_ops)
            .await
            .unwrap();
        let retrieved = verifier2.get_system_public_key().unwrap();
        assert!(retrieved.is_none());
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::security::Ed25519KeyPair;
    use serde_json::json;

    #[tokio::test]
    async fn test_message_signing_and_verification() {
        let signer_keypair = Ed25519KeyPair::generate().unwrap();
        let signer = MessageSigner::new(signer_keypair);
        let verifier = MessageVerifier::new(60);

        let public_key_base64 = signer.keypair.public_key_base64();
        let key_info = PublicKeyInfo::new(
            SINGLE_PUBLIC_KEY_ID.to_string(),
            public_key_base64,
            "test_owner".to_string(),
            vec!["read".to_string()],
        );
        verifier.register_system_public_key(key_info).await.unwrap();

        let payload = json!({"data": "hello world"});
        let signed_message = signer.sign_message(payload).unwrap();

        let result = verifier.verify_message(&signed_message).unwrap();
        assert!(result.is_valid);
        assert!(result.timestamp_valid);
        assert_eq!(
            result.public_key_info.unwrap().id,
            SINGLE_PUBLIC_KEY_ID.to_string()
        );
    }

    #[tokio::test]
    async fn test_permission_verification() {
        let signer_keypair = Ed25519KeyPair::generate().unwrap();
        let signer = MessageSigner::new(signer_keypair);
        let verifier = MessageVerifier::new(60);

        let public_key_base64 = signer.keypair.public_key_base64();
        let key_info = PublicKeyInfo::new(
            SINGLE_PUBLIC_KEY_ID.to_string(),
            public_key_base64,
            "test_owner".to_string(),
            vec!["read".to_string()],
        );
        verifier.register_system_public_key(key_info).await.unwrap();

        let payload = json!({"action": "read_data"});
        let signed_message = signer.sign_message(payload).unwrap();

        // Test with sufficient permissions
        let result1 = verifier
            .verify_message_with_permissions(&signed_message, &["read".to_string()])
            .unwrap();
        assert!(result1.is_valid);

        // Test with insufficient permissions
        let result2 = verifier
            .verify_message_with_permissions(&signed_message, &["write".to_string()])
            .unwrap();
        assert!(!result2.is_valid);
        assert!(result2
            .error
            .unwrap()
            .contains("Missing required permission"));
    }

    #[tokio::test]
    async fn test_timestamp_validation() {
        let signer_keypair = Ed25519KeyPair::generate().unwrap();
        let signer = MessageSigner::new(signer_keypair);
        let verifier = MessageVerifier::new(5); // 5 second tolerance

        let public_key_base64 = signer.keypair.public_key_base64();
        let key_info = PublicKeyInfo::new(
            "test_key".to_string(),
            public_key_base64,
            "test_owner".to_string(),
            vec![],
        );
        verifier.register_system_public_key(key_info).await.unwrap();

        // Message with valid timestamp
        let valid_payload = json!({"msg": "valid"});
        let valid_message = signer.sign_message(valid_payload).unwrap();
        let valid_result = verifier.verify_message(&valid_message).unwrap();
        assert!(valid_result.is_valid);
        assert!(valid_result.timestamp_valid);

        // Message with expired timestamp
        let expired_payload = json!({"msg": "expired"});
        let mut expired_message = signer.sign_message(expired_payload).unwrap();
        expired_message.timestamp -= 10; // 10 seconds ago, outside tolerance
        let expired_result = verifier.verify_message(&expired_message).unwrap();
        assert!(!expired_result.timestamp_valid);
    }
}