matchy 2.0.1

Fast database for IP address and pattern matching with rich data storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341

// Test for sequential builder bug
//
// Bug: When creating multiple builders sequentially (create → add → save → free → repeat),
// data from previous builders incorrectly appears in subsequent builders.

use matchy::database::QueryResult;
use matchy::{Database, DatabaseBuilder};
use matchy_match_mode::MatchMode;
use std::collections::HashMap;
use tempfile::tempdir;

/// Helper to check if a lookup result is an actual match (not NotFound)
fn is_match(result: &Option<QueryResult>) -> bool {
    match result {
        Some(QueryResult::NotFound) => false,
        Some(_) => true,
        None => false,
    }
}

/// Test that multiple sequential builders don't share state
#[test]
fn test_sequential_builders_isolated() {
    let temp_dir = tempdir().expect("Failed to create temp dir");
    let db1_path = temp_dir.path().join("db1.mxy");
    let db2_path = temp_dir.path().join("db2.mxy");
    let db3_path = temp_dir.path().join("db3.mxy");

    // Builder 1: Add IP 1.1.1.1 with data {"name": "first"}
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "name".to_string(),
            matchy_data_format::DataValue::String("first".to_string()),
        );
        builder.add_entry("1.1.1.1", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db1_path, bytes).unwrap();
    }

    // Builder 2: Add IP 2.2.2.2 with data {"name": "second"}
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "name".to_string(),
            matchy_data_format::DataValue::String("second".to_string()),
        );
        builder.add_entry("2.2.2.2", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db2_path, bytes).unwrap();
    }

    // Builder 3: Empty database (no entries)
    {
        let builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let bytes = builder.build().unwrap();
        std::fs::write(&db3_path, bytes).unwrap();
    }

    // Now verify the databases contain the correct data

    // DB1 should have 1.1.1.1 -> "first"
    let db1 = Database::from(db1_path.to_str().unwrap())
        .open()
        .expect("Failed to open db1");
    let result1 = db1.lookup("1.1.1.1").expect("lookup failed");
    assert!(is_match(&result1), "DB1 should find 1.1.1.1");

    // DB1 should NOT have 2.2.2.2
    let result1_other = db1.lookup("2.2.2.2").expect("lookup failed");
    assert!(!is_match(&result1_other), "DB1 should NOT find 2.2.2.2");

    // DB2 should have 2.2.2.2 -> "second"
    let db2 = Database::from(db2_path.to_str().unwrap())
        .open()
        .expect("Failed to open db2");
    let result2 = db2.lookup("2.2.2.2").expect("lookup failed");
    assert!(is_match(&result2), "DB2 should find 2.2.2.2");

    // DB2 should NOT have 1.1.1.1
    let result2_other = db2.lookup("1.1.1.1").expect("lookup failed");
    assert!(!is_match(&result2_other), "DB2 should NOT find 1.1.1.1");

    // DB3 should be empty - should NOT find 1.1.1.1 (the bug would show this)
    let db3 = Database::from(db3_path.to_str().unwrap())
        .open()
        .expect("Failed to open db3");
    let result3 = db3.lookup("1.1.1.1").expect("lookup failed");
    assert!(
        !is_match(&result3),
        "DB3 (empty) should NOT find 1.1.1.1 - BUG: data leaked from builder 1!"
    );
    let result3_other = db3.lookup("2.2.2.2").expect("lookup failed");
    assert!(
        !is_match(&result3_other),
        "DB3 (empty) should NOT find 2.2.2.2"
    );
}

/// Test the C API pattern more closely - simulating save-then-reuse pattern
/// The bug description suggests data gets "shifted" by one builder
#[test]
fn test_sequential_builders_c_api_pattern() {
    let temp_dir = tempdir().expect("Failed to create temp dir");

    // This test simulates what happens in the C API:
    // 1. Create builder
    // 2. Add entries
    // 3. Save (which replaces internal builder with empty one)
    // 4. Free builder
    // 5. Create new builder
    // 6. Add different entries
    // 7. Save
    // etc.

    let db1_path = temp_dir.path().join("capi_db1.mxy");
    let db2_path = temp_dir.path().join("capi_db2.mxy");
    let db3_path = temp_dir.path().join("capi_db3.mxy");

    // Simulate C API matchy_builder_save behavior:
    // It does std::mem::replace to take ownership, leaving an empty builder behind

    // Builder 1
    let mut builder1 = DatabaseBuilder::new(MatchMode::CaseSensitive);
    let mut data1 = HashMap::new();
    data1.insert(
        "name".to_string(),
        matchy_data_format::DataValue::String("first".to_string()),
    );
    builder1.add_entry("1.1.1.1", data1).unwrap();

    // Simulate matchy_builder_save: take ownership via replace
    let builder_to_build = std::mem::replace(
        &mut builder1,
        DatabaseBuilder::new(MatchMode::CaseSensitive),
    );
    let bytes1 = builder_to_build.build().unwrap();
    std::fs::write(&db1_path, &bytes1).unwrap();
    // builder1 is now empty (the replacement)
    drop(builder1); // Simulate matchy_builder_free

    // Builder 2
    let mut builder2 = DatabaseBuilder::new(MatchMode::CaseSensitive);
    let mut data2 = HashMap::new();
    data2.insert(
        "name".to_string(),
        matchy_data_format::DataValue::String("second".to_string()),
    );
    builder2.add_entry("2.2.2.2", data2).unwrap();

    let builder_to_build = std::mem::replace(
        &mut builder2,
        DatabaseBuilder::new(MatchMode::CaseSensitive),
    );
    let bytes2 = builder_to_build.build().unwrap();
    std::fs::write(&db2_path, &bytes2).unwrap();
    drop(builder2);

    // Builder 3 - empty
    let mut builder3 = DatabaseBuilder::new(MatchMode::CaseSensitive);
    let builder_to_build = std::mem::replace(
        &mut builder3,
        DatabaseBuilder::new(MatchMode::CaseSensitive),
    );
    let bytes3 = builder_to_build.build().unwrap();
    std::fs::write(&db3_path, &bytes3).unwrap();
    drop(builder3);

    // Verify
    let db1 = Database::from(db1_path.to_str().unwrap()).open().unwrap();
    let db2 = Database::from(db2_path.to_str().unwrap()).open().unwrap();
    let db3 = Database::from(db3_path.to_str().unwrap()).open().unwrap();

    // DB1: 1.1.1.1 should be found
    assert!(
        is_match(&db1.lookup("1.1.1.1").unwrap()),
        "DB1 should have 1.1.1.1"
    );
    assert!(
        !is_match(&db1.lookup("2.2.2.2").unwrap()),
        "DB1 should NOT have 2.2.2.2"
    );

    // DB2: 2.2.2.2 should be found (BUG: might be empty if data shifted)
    assert!(
        is_match(&db2.lookup("2.2.2.2").unwrap()),
        "DB2 should have 2.2.2.2 - BUG if missing!"
    );
    assert!(
        !is_match(&db2.lookup("1.1.1.1").unwrap()),
        "DB2 should NOT have 1.1.1.1"
    );

    // DB3: should be empty (BUG: might have 1.1.1.1 if data shifted)
    assert!(
        !is_match(&db3.lookup("1.1.1.1").unwrap()),
        "DB3 should NOT have 1.1.1.1 - BUG: builder 1's data leaked!"
    );
    assert!(
        !is_match(&db3.lookup("2.2.2.2").unwrap()),
        "DB3 should NOT have 2.2.2.2"
    );
}

/// Test interleaved queries on multiple databases
/// This tests the scenario that would cause cache thrashing with the old design
#[test]
fn test_interleaved_queries_multiple_databases() {
    let temp_dir = tempdir().expect("Failed to create temp dir");
    let db1_path = temp_dir.path().join("interleave_db1.mxy");
    let db2_path = temp_dir.path().join("interleave_db2.mxy");

    // Build DB1: contains 1.1.1.1
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "source".to_string(),
            matchy_data_format::DataValue::String("db1".to_string()),
        );
        builder.add_entry("1.1.1.1", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db1_path, bytes).unwrap();
    }

    // Build DB2: contains 1.1.1.1 with DIFFERENT data
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "source".to_string(),
            matchy_data_format::DataValue::String("db2".to_string()),
        );
        builder.add_entry("1.1.1.1", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db2_path, bytes).unwrap();
    }

    // Open both databases
    let db1 = Database::from(db1_path.to_str().unwrap()).open().unwrap();
    let db2 = Database::from(db2_path.to_str().unwrap()).open().unwrap();

    // Interleave queries - same query string, different databases
    // Each should return its own data, not cached data from the other
    for _ in 0..10 {
        let r1 = db1.lookup("1.1.1.1").unwrap();
        let r2 = db2.lookup("1.1.1.1").unwrap();

        // Extract the "source" field from each result
        let source1 = match &r1 {
            Some(QueryResult::Ip { data, .. }) => match data {
                matchy_data_format::DataValue::Map(m) => match m.get("source") {
                    Some(matchy_data_format::DataValue::String(s)) => s.as_str(),
                    _ => panic!("Expected source string in db1 result"),
                },
                _ => panic!("Expected map in db1 result"),
            },
            _ => panic!("Expected Ip result from db1"),
        };

        let source2 = match &r2 {
            Some(QueryResult::Ip { data, .. }) => match data {
                matchy_data_format::DataValue::Map(m) => match m.get("source") {
                    Some(matchy_data_format::DataValue::String(s)) => s.as_str(),
                    _ => panic!("Expected source string in db2 result"),
                },
                _ => panic!("Expected map in db2 result"),
            },
            _ => panic!("Expected Ip result from db2"),
        };

        assert_eq!(
            source1, "db1",
            "DB1 query returned wrong data (cache collision!)"
        );
        assert_eq!(
            source2, "db2",
            "DB2 query returned wrong data (cache collision!)"
        );
    }
}

/// Test with patterns (globs) to ensure the bug affects all entry types
#[test]
fn test_sequential_builders_with_patterns() {
    let temp_dir = tempdir().expect("Failed to create temp dir");
    let db1_path = temp_dir.path().join("pattern_db1.mxy");
    let db2_path = temp_dir.path().join("pattern_db2.mxy");

    // Builder 1: pattern "*.evil.com"
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "category".to_string(),
            matchy_data_format::DataValue::String("malware".to_string()),
        );
        builder.add_entry("*.evil.com", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db1_path, bytes).unwrap();
    }

    // Builder 2: pattern "*.good.com"
    {
        let mut builder = DatabaseBuilder::new(MatchMode::CaseSensitive);
        let mut data = HashMap::new();
        data.insert(
            "category".to_string(),
            matchy_data_format::DataValue::String("safe".to_string()),
        );
        builder.add_entry("*.good.com", data).unwrap();
        let bytes = builder.build().unwrap();
        std::fs::write(&db2_path, bytes).unwrap();
    }

    // Verify
    let db1 = Database::from(db1_path.to_str().unwrap()).open().unwrap();
    let db2 = Database::from(db2_path.to_str().unwrap()).open().unwrap();

    // DB1 should match *.evil.com
    assert!(
        is_match(&db1.lookup("test.evil.com").unwrap()),
        "DB1 should match *.evil.com"
    );
    assert!(
        !is_match(&db1.lookup("test.good.com").unwrap()),
        "DB1 should NOT match *.good.com"
    );

    // DB2 should match *.good.com
    assert!(
        is_match(&db2.lookup("test.good.com").unwrap()),
        "DB2 should match *.good.com"
    );
    assert!(
        !is_match(&db2.lookup("test.evil.com").unwrap()),
        "DB2 should NOT match *.evil.com - BUG if matched!"
    );
}