remdb 0.3.1

嵌入式内存数据库
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
use crate::try_lock;

use super::{CompressionType, LifecycleManager, PartitionManager, TimeSeriesIndex};
use crate::{RemDbError, Result, TableDef};
use alloc::{sync::Arc, vec::Vec};
use core::time::Duration;

#[cfg(feature = "std")]
use std::sync::Mutex;

#[cfg(not(feature = "std"))]
use crate::memory::allocator::Mutex;

/// 时序数据配置
#[derive(Debug, Clone, Copy)]
pub struct TimeSeriesConfig {
    /// 分区时长(秒)
    pub partition_duration_secs: u64,
    /// 数据保留期(秒)
    pub retention_period_secs: u64,
    /// 压缩类型
    pub compression: CompressionType,
    /// 最大分区数
    pub max_partitions: usize,
}

impl TimeSeriesConfig {
    /// 创建一个新的时序数据配置
    pub const fn new(
        partition_duration_secs: u64,
        retention_period_secs: u64,
        compression: CompressionType,
        max_partitions: usize,
    ) -> Self {
        Self {
            partition_duration_secs,
            retention_period_secs,
            compression,
            max_partitions,
        }
    }

    /// 获取分区时长
    pub fn partition_duration(&self) -> Duration {
        Duration::from_secs(self.partition_duration_secs)
    }

    /// 获取数据保留期
    pub fn retention_period(&self) -> Duration {
        Duration::from_secs(self.retention_period_secs)
    }
}

impl Default for TimeSeriesConfig {
    fn default() -> Self {
        Self::DEFAULT
    }
}

/// 时序数据配置默认值
pub const DEFAULT_TIME_SERIES_CONFIG: TimeSeriesConfig = TimeSeriesConfig::new(
    3600,          // 1小时
    7 * 24 * 3600, // 7天
    CompressionType::DeltaRunLength,
    1000,
);

impl TimeSeriesConfig {
    /// 时序数据配置默认值
    pub const DEFAULT: Self = DEFAULT_TIME_SERIES_CONFIG;
}

/// 时序表定义
#[derive(Debug)]
pub struct TimeSeriesTableDef {
    /// 基础表定义
    pub base: TableDef,
    /// 时间字段索引
    pub time_field: usize,
    /// 值字段索引
    pub value_field: usize,
    /// 标签字段索引列表
    pub tag_fields: Box<[usize]>,
    /// 时序数据配置
    pub config: TimeSeriesConfig,
}

/// 时序数据记录
#[derive(Debug, Clone, Copy)]
pub struct TimeSeriesRecord {
    /// 时间戳
    pub timestamp: u64,
    ///    pub value: f64,
    /// 标签数量
    pub tag_count: u8,
    /// 标签数据(可变长度)
    pub tags: [u64; 8], // 支持最多8个标签
}

/// 预聚合数据配置
#[derive(Debug, Clone, PartialEq)]
pub struct PreAggregationConfig {
    /// 预聚合时间间隔(秒)
    pub interval_seconds: u64,
    /// 预聚合函数
    pub aggregation: String,
}

/// 预聚合数据存储
pub struct PreAggregationStore {
    /// 预聚合配置
    pub configs: Vec<PreAggregationConfig>,
    /// 预聚合数据(按时间间隔和标签组合存储)
    pub data: std::collections::HashMap<(u64, u64), f64>, // (time_bucket, tag_hash) -> aggregated_value
}

/// 时序表结构
pub struct TimeSeriesTable {
    /// 表定义
    pub def: Arc<TimeSeriesTableDef>,
    /// 分区管理器
    pub partitions: Arc<Mutex<PartitionManager>>,
    /// 时序索引
    pub index: Arc<TimeSeriesIndex>,
    /// 生命周期管理器
    pub lifecycle: LifecycleManager,
    /// 预聚合数据存储
    pub pre_aggregation: Arc<Mutex<PreAggregationStore>>,
}

impl TimeSeriesTable {
    /// 创建新的时序表
    pub fn new(def: Arc<TimeSeriesTableDef>, index: Arc<TimeSeriesIndex>) -> Result<Self> {
        // 检查时间字段和值字段的有效性
        if def.time_field >= def.base.fields.len() {
            return Err(RemDbError::FieldNotFound);
        }

        if def.value_field >= def.base.fields.len() {
            return Err(RemDbError::FieldNotFound);
        }

        // 检查标签字段的有效性
        for tag_field in def.tag_fields.iter() {
            if *tag_field >= def.base.fields.len() {
                return Err(RemDbError::FieldNotFound);
            }
        }

        // 创建分区管理器
        let partition_manager = Arc::new(Mutex::new(PartitionManager::new(
            def.config.partition_duration(),
            def.config.max_partitions,
        )));

        // 创建生命周期管理器
        let mut lifecycle_manager = LifecycleManager::new(def.config.retention_period());

        // 设置清理闭包
        let partitions_clone = partition_manager.clone();
        let retention_period = def.config.retention_period();
        lifecycle_manager.set_cleanup_callback(move || {
            let mut partitions_guard = try_lock!(partitions_clone);
            let current_time = LifecycleManager::get_current_timestamp();
            partitions_guard.cleanup_expired_partitions(current_time, retention_period);
        });

        // 创建预聚合数据存储
        let pre_aggregation = Arc::new(Mutex::new(PreAggregationStore {
            configs: Vec::new(),
            data: std::collections::HashMap::new(),
        }));

        Ok(Self {
            def,
            partitions: partition_manager,
            index,
            lifecycle: lifecycle_manager,
            pre_aggregation,
        })
    }

    /// 添加预聚合配置
    pub fn add_pre_aggregation(&self, interval_seconds: u64, aggregation: &str) -> Result<()> {
        let mut pre_aggregation_guard = try_lock!(self.pre_aggregation);

        // 检查是否已存在相同配置
        let existing_config = pre_aggregation_guard.configs.iter().find(|config| {
            config.interval_seconds == interval_seconds && config.aggregation == aggregation
        });

        if existing_config.is_some() {
            return Ok(()); // 配置已存在,无需重复添加
        }

        // 添加新的预聚合配置
        pre_aggregation_guard.configs.push(PreAggregationConfig {
            interval_seconds,
            aggregation: aggregation.to_string(),
        });

        Ok(())
    }

    /// 使用预聚合数据执行查询
    pub fn query_pre_aggregated(
        &self,
        start_time: u64,
        end_time: u64,
        interval_seconds: u64,
        aggregation: &str,
    ) -> Result<Vec<TimeSeriesRecord>> {
        let pre_aggregation_guard = try_lock!(self.pre_aggregation);

        // 检查预聚合配置是否存在
        let config_exists = pre_aggregation_guard.configs.iter().any(|config| {
            config.interval_seconds == interval_seconds && config.aggregation == aggregation
        });

        if !config_exists {
            return Err(RemDbError::ConfigError); // 预聚合配置不存在
        }

        // 计算时间桶范围
        let interval_nanos = interval_seconds * 1_000_000_000u64;
        let start_bucket = start_time / interval_nanos;
        let end_bucket = end_time / interval_nanos;

        // 收集预聚合数据
        let mut result = Vec::new();

        for bucket in start_bucket..=end_bucket {
            // 查找该时间桶的所有预聚合数据
            for ((stored_bucket, _tag_hash), value) in pre_aggregation_guard.data.iter() {
                if *stored_bucket == bucket {
                    // 构建时序记录
                    result.push(TimeSeriesRecord {
                        timestamp: bucket * interval_nanos,
                        value: *value,
                        tag_count: 0, // 简化处理,实际应该从tag_hash恢复标签
                        tags: [0; 8],
                    });
                }
            }
        }

        Ok(result)
    }

    /// 更新预聚合数据
    fn update_pre_aggregations(&self, record: &TimeSeriesRecord) {
        let mut pre_aggregation_guard = try_lock!(self.pre_aggregation);

        // 先复制配置,避免借用冲突
        let configs = pre_aggregation_guard.configs.clone();

        // 为每个预聚合配置更新数据
        for config in &configs {
            let interval_nanos = config.interval_seconds * 1_000_000_000u64;
            let time_bucket = record.timestamp / interval_nanos;

            // 计算标签哈希(简化处理)
            let tag_hash = record.tag_count as u64; // 实际应该基于标签值计算哈希

            let key = (time_bucket, tag_hash);

            // 根据聚合函数更新值
            match config.aggregation.as_str() {
                "avg" => {
                    // 平均值需要跟踪总和和计数,这里简化处理
                    let current_value = *pre_aggregation_guard.data.get(&key).unwrap_or(&0.0);
                    // 简化实现:使用移动平均
                    let new_value = (current_value + record.value) / 2.0;
                    pre_aggregation_guard.data.insert(key, new_value);
                }
                "sum" => {
                    let current_value = *pre_aggregation_guard.data.get(&key).unwrap_or(&0.0);
                    pre_aggregation_guard
                        .data
                        .insert(key, current_value + record.value);
                }
                "min" => {
                    let current_value = *pre_aggregation_guard.data.get(&key).unwrap_or(&f64::MAX);
                    pre_aggregation_guard
                        .data
                        .insert(key, f64::min(current_value, record.value));
                }
                "max" => {
                    let current_value = *pre_aggregation_guard.data.get(&key).unwrap_or(&f64::MIN);
                    pre_aggregation_guard
                        .data
                        .insert(key, f64::max(current_value, record.value));
                }
                _ => {}
            }
        }
    }

    /// 批量写入时序数据
    pub unsafe fn batch_write(
        &mut self,
        records: *const TimeSeriesRecord,
        count: usize,
    ) -> Result<usize> {
        if records.is_null() || count == 0 {
            return Err(RemDbError::ConfigError);
        }

        let mut inserted = 0;

        // 遍历所有记录,写入到对应的分区
        for i in 0..count {
            let record = *records.add(i);

            // 获取或创建分区
            let mut partitions_guard = try_lock!(self.partitions);
            let partition = partitions_guard.get_or_create_partition(record.timestamp);

            // 写入记录到分区
            let mut partition_guard = try_lock!(partition);
            partition_guard.records.push(record);
            partition_guard.stats.record_count += 1;

            // 更新索引
            self.index.insert(record.timestamp, inserted as usize);

            // 更新预聚合数据
            self.update_pre_aggregations(&record);

            inserted += 1;
        }

        Ok(inserted)
    }

    /// 事务化批量写入时序数据
    /// 确保一批数据要么全部成功插入并立即可见,要么全部回滚
    pub fn write_timeseries_batch(&mut self, data_points: &[TimeSeriesRecord]) -> Result<usize> {
        if data_points.is_empty() {
            return Err(RemDbError::ConfigError);
        }

        // 检查是否有活跃事务
        let has_active_tx = crate::transaction::has_active_tx();

        // 如果没有活跃事务,返回错误,要求调用者显式开始事务
        // 这是为了简化实现,避免直接访问Transaction结构体的私有字段
        if !has_active_tx {
            return Err(RemDbError::TransactionError);
        }

        let mut inserted = 0;
        let table_id = self.def.base.id;

        // 批量写入逻辑
        for (i, record) in data_points.iter().enumerate() {
            // 获取或创建分区
            let mut partitions_guard = try_lock!(self.partitions);
            let partition = partitions_guard.get_or_create_partition(record.timestamp);

            // 写入记录到分区
            let mut partition_guard = try_lock!(partition);
            partition_guard.records.push(*record);
            partition_guard.stats.record_count = partition_guard.records.len();

            // 更新索引
            self.index.insert(record.timestamp, inserted as usize);

            // 更新预聚合数据
            self.update_pre_aggregations(record);

            // 记录事务日志
            unsafe {
                // 获取当前事务
                if let Some(mut tx_ptr) = crate::transaction::get_current_tx() {
                    let tx_mut = tx_ptr.as_mut();

                    // 添加日志项
                    let data_size = core::mem::size_of::<TimeSeriesRecord>();
                    let tx_id = tx_mut.id;
                    let record_slice =
                        core::slice::from_raw_parts(record as *const _ as *const u8, data_size);
                    tx_mut.begin_log_item(
                        tx_id,
                        crate::transaction::LogOperation::TimeSeriesInsert,
                        table_id,
                        i as u16, // 使用索引作为record_id
                        data_size as u16,
                        None,               // 旧数据为null
                        Some(record_slice), // 新数据指针
                    );
                }
            }

            inserted += 1;
        }

        Ok(inserted)
    }

    /// 时间范围查询
    pub fn query_time_range(
        &self,
        start_time: u64,
        end_time: u64,
    ) -> Result<Vec<TimeSeriesRecord>> {
        // 获取所有相关分区
        let partitions_guard = try_lock!(self.partitions);
        let relevant_partitions = partitions_guard.get_partitions_in_range(start_time, end_time);

        let mut results = Vec::new();

        // 遍历所有相关分区,查询符合条件的记录
        for partition in relevant_partitions {
            let partition_guard = try_lock!(partition);

            // 遍历分区中的记录,过滤符合时间范围的记录
            for record in &partition_guard.records {
                if record.timestamp >= start_time && record.timestamp <= end_time {
                    results.push(*record);
                }
            }
        }

        Ok(results)
    }
}