datafold 0.1.55

A personal database for data sovereignty with AI-powered ingestion
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

use super::native_index_classification::{
    ClassificationRequest, ClassificationType, ExtractedEntity, FieldClassification, SplitStrategy,
};
use crate::ingestion::config::IngestionConfig;
use crate::schema::SchemaError;

pub struct NativeIndexAIClassifier {}

impl NativeIndexAIClassifier {
    pub fn new(_config: IngestionConfig) -> Self {
        Self {}
    }

    /// Classify a field using heuristics
    pub async fn classify_field(
        &self,
        request: ClassificationRequest,
    ) -> Result<FieldClassification, SchemaError> {
        // Use heuristic-based classification
        Ok(self.classify_field_heuristic(&request.field_name, &request.sample_values))
    }

    /// Heuristic-based field classification
    fn classify_field_heuristic(
        &self,
        field_name: &str,
        _sample_values: &[String],
    ) -> FieldClassification {
        let field_lower = field_name.to_lowercase();
        let mut classifications = Vec::new();
        let mut strategies = std::collections::HashMap::new();

        // Check for common field patterns
        if field_lower.contains("email") {
            classifications.push(ClassificationType::Email);
            strategies.insert(ClassificationType::Email, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("phone") || field_lower.contains("mobile") {
            classifications.push(ClassificationType::Phone);
            strategies.insert(ClassificationType::Phone, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("url")
            || field_lower.contains("link")
            || field_lower.contains("website")
        {
            classifications.push(ClassificationType::Url);
            strategies.insert(ClassificationType::Url, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("date")
            || field_lower.contains("time")
            || field_lower.contains("created")
            || field_lower.contains("updated")
        {
            classifications.push(ClassificationType::Date);
            strategies.insert(ClassificationType::Date, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("name")
            || field_lower.contains("author")
            || field_lower.contains("user")
        {
            classifications.push(ClassificationType::NamePerson);
            strategies.insert(ClassificationType::NamePerson, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("company") || field_lower.contains("organization") {
            classifications.push(ClassificationType::NameCompany);
            strategies.insert(ClassificationType::NameCompany, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("location")
            || field_lower.contains("city")
            || field_lower.contains("country")
            || field_lower.contains("place")
        {
            classifications.push(ClassificationType::NamePlace);
            strategies.insert(ClassificationType::NamePlace, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("tag") || field_lower.contains("hashtag") {
            classifications.push(ClassificationType::Hashtag);
            strategies.insert(ClassificationType::Hashtag, SplitStrategy::KeepWhole);
        }

        if field_lower.contains("username") || field_lower.contains("handle") {
            classifications.push(ClassificationType::Username);
            strategies.insert(ClassificationType::Username, SplitStrategy::KeepWhole);
        }

        // Always add word classification for general text search
        classifications.push(ClassificationType::Word);
        strategies.insert(ClassificationType::Word, SplitStrategy::SplitWords);

        FieldClassification {
            field_name: field_name.to_string(),
            classifications,
            strategies,
            entities: Vec::new(),
            cacheable: true,
        }
    }

    /// Extract entities from value
    pub async fn extract_entities_from_value(
        &self,
        _value: &str,
        _classification: &ClassificationType,
    ) -> Result<Vec<ExtractedEntity>, SchemaError> {
        Ok(Vec::new())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_heuristic_classification_email() {
        let config = IngestionConfig::default();
        let classifier = NativeIndexAIClassifier::new(config);

        let result = classifier.classify_field_heuristic("email", &[]);

        assert!(result.has_classification(&ClassificationType::Email));
        assert!(result.has_classification(&ClassificationType::Word));
    }

    #[test]
    fn test_heuristic_classification_author() {
        let config = IngestionConfig::default();
        let classifier = NativeIndexAIClassifier::new(config);

        let result = classifier.classify_field_heuristic("author", &[]);

        assert!(result.has_classification(&ClassificationType::NamePerson));
        assert!(result.has_classification(&ClassificationType::Word));
    }
}