1use std::collections::HashSet;
2
3use chrono::{DateTime, Utc};
4use cognee_utils::tracing_keys::{COGNEE_DB_ROW_COUNT, COGNEE_DB_SYSTEM};
5use sea_orm::sea_query::{Alias, Expr, OnConflict, Query};
6use sea_orm::{
7 ColumnTrait, Condition, ConnectionTrait, DatabaseConnection, EntityTrait, PaginatorTrait,
8 QueryFilter, QueryOrder, QuerySelect, TransactionTrait,
9};
10use tracing::{Span, instrument};
11use uuid::Uuid;
12
13use crate::conversions::map_sea_err;
14use crate::database_system_label;
15use crate::entities::{edge, node};
16use crate::types::{DatabaseError, GraphEdge, GraphNode};
17use crate::uuid_hex;
18
19const PROVENANCE_INSERT_BATCH: usize = 500;
27
28fn dedup_keeping_last<T, F>(items: &[T], key: F) -> Vec<&T>
41where
42 F: Fn(&T) -> Uuid,
43{
44 let mut seen: HashSet<Uuid> = HashSet::with_capacity(items.len());
46 let mut out: Vec<&T> = Vec::with_capacity(items.len());
47 for item in items.iter().rev() {
48 if seen.insert(key(item)) {
49 out.push(item);
50 }
51 }
52 out.reverse();
53 out
54}
55
56#[instrument(
68 name = "cognee.db.relational.graph_storage.upsert_nodes",
69 level = "info",
70 skip_all,
71 fields(cognee.db.system = tracing::field::Empty),
72 err,
73)]
74pub async fn upsert_nodes(
75 db: &DatabaseConnection,
76 nodes: &[GraphNode],
77) -> Result<(), DatabaseError> {
78 upsert_nodes_on(db, nodes).await
79}
80
81async fn upsert_nodes_on<C: ConnectionTrait>(
82 db: &C,
83 nodes: &[GraphNode],
84) -> Result<(), DatabaseError> {
85 Span::current().record(COGNEE_DB_SYSTEM, crate::connection_system_label(db));
86 if nodes.is_empty() {
87 return Ok(());
88 }
89 for batch in nodes.chunks(PROVENANCE_INSERT_BATCH) {
91 let models: Vec<node::ActiveModel> = dedup_keeping_last(batch, |n| n.id)
94 .into_iter()
95 .map(node::ActiveModel::from)
96 .collect();
97 node::Entity::insert_many(models)
98 .on_conflict(
99 OnConflict::column(node::Column::Id)
100 .update_columns([
101 node::Column::Slug,
102 node::Column::UserId,
103 node::Column::DataId,
104 node::Column::DatasetId,
105 node::Column::Label,
106 node::Column::NodeType,
107 node::Column::IndexedFields,
108 node::Column::Attributes,
109 ])
110 .to_owned(),
111 )
112 .exec(db)
113 .await
114 .map_err(map_sea_err)?;
115 }
116 Ok(())
117}
118
119#[instrument(
120 name = "cognee.db.relational.graph_storage.get_nodes_by_dataset",
121 level = "info",
122 skip_all,
123 fields(
124 cognee.db.system = tracing::field::Empty,
125 cognee.db.row_count = tracing::field::Empty,
126 ),
127 err,
128)]
129pub async fn get_nodes_by_dataset(
130 db: &DatabaseConnection,
131 dataset_id: Uuid,
132) -> Result<Vec<GraphNode>, DatabaseError> {
133 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
134 let rows: Vec<GraphNode> = node::Entity::find()
135 .filter(node::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id)))
136 .order_by_asc(node::Column::CreatedAt)
137 .all(db)
138 .await
139 .map_err(map_sea_err)?
140 .into_iter()
141 .map(GraphNode::from)
142 .collect();
143 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
144 Ok(rows)
145}
146
147#[instrument(
148 name = "cognee.db.relational.graph_storage.delete_nodes_by_data",
149 level = "info",
150 skip_all,
151 fields(cognee.db.system = tracing::field::Empty),
152 err,
153)]
154pub async fn delete_nodes_by_data(
155 db: &DatabaseConnection,
156 data_id: Uuid,
157) -> Result<(), DatabaseError> {
158 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
159 node::Entity::delete_many()
160 .filter(node::Column::DataId.eq(uuid_hex::to_hex(data_id)))
161 .exec(db)
162 .await
163 .map_err(map_sea_err)?;
164 Ok(())
165}
166
167#[instrument(
175 name = "cognee.db.relational.graph_storage.upsert_edges",
176 level = "info",
177 skip_all,
178 fields(cognee.db.system = tracing::field::Empty),
179 err,
180)]
181pub async fn upsert_edges(
182 db: &DatabaseConnection,
183 edges: &[GraphEdge],
184) -> Result<(), DatabaseError> {
185 upsert_edges_on(db, edges).await
186}
187
188async fn upsert_edges_on<C: ConnectionTrait>(
189 db: &C,
190 edges: &[GraphEdge],
191) -> Result<(), DatabaseError> {
192 Span::current().record(COGNEE_DB_SYSTEM, crate::connection_system_label(db));
193 if edges.is_empty() {
194 return Ok(());
195 }
196 for batch in edges.chunks(PROVENANCE_INSERT_BATCH) {
198 let models: Vec<edge::ActiveModel> = dedup_keeping_last(batch, |e| e.id)
201 .into_iter()
202 .map(edge::ActiveModel::from)
203 .collect();
204 edge::Entity::insert_many(models)
205 .on_conflict(
206 OnConflict::column(edge::Column::Id)
207 .update_columns([
208 edge::Column::Slug,
209 edge::Column::UserId,
210 edge::Column::DataId,
211 edge::Column::DatasetId,
212 edge::Column::SourceNodeId,
213 edge::Column::DestinationNodeId,
214 edge::Column::RelationshipName,
215 edge::Column::Label,
216 edge::Column::Attributes,
217 ])
218 .to_owned(),
219 )
220 .exec(db)
221 .await
222 .map_err(map_sea_err)?;
223 }
224 Ok(())
225}
226
227#[instrument(
244 name = "cognee.db.relational.graph_storage.upsert_provenance_graph",
245 level = "info",
246 skip_all,
247 fields(cognee.db.system = tracing::field::Empty),
248 err,
249)]
250pub async fn upsert_provenance_graph(
251 db: &DatabaseConnection,
252 nodes: &[GraphNode],
253 edges: &[GraphEdge],
254) -> Result<(), DatabaseError> {
255 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
256 if nodes.is_empty() && edges.is_empty() {
257 return Ok(());
258 }
259 let txn = db.begin().await.map_err(map_sea_err)?;
260 upsert_nodes_on(&txn, nodes).await?;
261 upsert_edges_on(&txn, edges).await?;
262 txn.commit().await.map_err(map_sea_err)?;
263 Ok(())
264}
265
266#[instrument(
267 name = "cognee.db.relational.graph_storage.get_edges_by_dataset",
268 level = "info",
269 skip_all,
270 fields(
271 cognee.db.system = tracing::field::Empty,
272 cognee.db.row_count = tracing::field::Empty,
273 ),
274 err,
275)]
276pub async fn get_edges_by_dataset(
277 db: &DatabaseConnection,
278 dataset_id: Uuid,
279) -> Result<Vec<GraphEdge>, DatabaseError> {
280 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
281 let rows: Vec<GraphEdge> = edge::Entity::find()
282 .filter(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id)))
283 .order_by_asc(edge::Column::CreatedAt)
284 .all(db)
285 .await
286 .map_err(map_sea_err)?
287 .into_iter()
288 .map(GraphEdge::from)
289 .collect();
290 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
291 Ok(rows)
292}
293
294#[instrument(
300 name = "cognee.db.relational.graph_storage.get_edges_since",
301 level = "info",
302 skip_all,
303 fields(
304 cognee.db.system = tracing::field::Empty,
305 cognee.db.row_count = tracing::field::Empty,
306 ),
307 err,
308)]
309pub async fn get_edges_since(
310 db: &DatabaseConnection,
311 dataset_id: Uuid,
312 since: Option<DateTime<Utc>>,
313 limit: u64,
314) -> Result<Vec<GraphEdge>, DatabaseError> {
315 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
316 let mut q = edge::Entity::find()
317 .filter(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id)))
318 .order_by_asc(edge::Column::CreatedAt)
319 .limit(limit);
320 if let Some(ts) = since {
321 q = q.filter(edge::Column::CreatedAt.gt(ts));
322 }
323 let rows: Vec<GraphEdge> = q
324 .all(db)
325 .await
326 .map_err(map_sea_err)?
327 .into_iter()
328 .map(GraphEdge::from)
329 .collect();
330 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
331 Ok(rows)
332}
333
334#[instrument(
337 name = "cognee.db.relational.graph_storage.get_nodes_by_ids",
338 level = "info",
339 skip_all,
340 fields(
341 cognee.db.system = tracing::field::Empty,
342 cognee.db.row_count = tracing::field::Empty,
343 ),
344 err,
345)]
346pub async fn get_nodes_by_ids(
347 db: &DatabaseConnection,
348 ids: &[String],
349) -> Result<Vec<GraphNode>, DatabaseError> {
350 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
351 if ids.is_empty() {
352 Span::current().record(COGNEE_DB_ROW_COUNT, 0i64);
353 return Ok(Vec::new());
354 }
355 let rows: Vec<GraphNode> = node::Entity::find()
356 .filter(node::Column::Id.is_in(ids.to_vec()))
357 .all(db)
358 .await
359 .map_err(map_sea_err)?
360 .into_iter()
361 .map(GraphNode::from)
362 .collect();
363 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
364 Ok(rows)
365}
366
367#[instrument(
368 name = "cognee.db.relational.graph_storage.delete_edges_by_data",
369 level = "info",
370 skip_all,
371 fields(cognee.db.system = tracing::field::Empty),
372 err,
373)]
374pub async fn delete_edges_by_data(
375 db: &DatabaseConnection,
376 data_id: Uuid,
377) -> Result<(), DatabaseError> {
378 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
379 edge::Entity::delete_many()
380 .filter(edge::Column::DataId.eq(uuid_hex::to_hex(data_id)))
381 .exec(db)
382 .await
383 .map_err(map_sea_err)?;
384 Ok(())
385}
386
387#[instrument(
393 name = "cognee.db.relational.graph_storage.get_nodes_by_data",
394 level = "info",
395 skip_all,
396 fields(
397 cognee.db.system = tracing::field::Empty,
398 cognee.db.row_count = tracing::field::Empty,
399 ),
400 err,
401)]
402pub async fn get_nodes_by_data(
403 db: &DatabaseConnection,
404 data_id: Uuid,
405 dataset_id: Uuid,
406) -> Result<Vec<GraphNode>, DatabaseError> {
407 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
408 let rows: Vec<GraphNode> = node::Entity::find()
409 .filter(
410 Condition::all()
411 .add(node::Column::DataId.eq(uuid_hex::to_hex(data_id)))
412 .add(node::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
413 )
414 .order_by_asc(node::Column::CreatedAt)
415 .all(db)
416 .await
417 .map_err(map_sea_err)?
418 .into_iter()
419 .map(GraphNode::from)
420 .collect();
421 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
422 Ok(rows)
423}
424
425#[instrument(
427 name = "cognee.db.relational.graph_storage.get_edges_by_data",
428 level = "info",
429 skip_all,
430 fields(
431 cognee.db.system = tracing::field::Empty,
432 cognee.db.row_count = tracing::field::Empty,
433 ),
434 err,
435)]
436pub async fn get_edges_by_data(
437 db: &DatabaseConnection,
438 data_id: Uuid,
439 dataset_id: Uuid,
440) -> Result<Vec<GraphEdge>, DatabaseError> {
441 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
442 let rows: Vec<GraphEdge> = edge::Entity::find()
443 .filter(
444 Condition::all()
445 .add(edge::Column::DataId.eq(uuid_hex::to_hex(data_id)))
446 .add(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
447 )
448 .order_by_asc(edge::Column::CreatedAt)
449 .all(db)
450 .await
451 .map_err(map_sea_err)?
452 .into_iter()
453 .map(GraphEdge::from)
454 .collect();
455 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
456 Ok(rows)
457}
458
459#[instrument(
465 name = "cognee.db.relational.graph_storage.delete_nodes_by_dataset",
466 level = "info",
467 skip_all,
468 fields(cognee.db.system = tracing::field::Empty),
469 err,
470)]
471pub async fn delete_nodes_by_dataset(
472 db: &DatabaseConnection,
473 dataset_id: Uuid,
474) -> Result<(), DatabaseError> {
475 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
476 node::Entity::delete_many()
477 .filter(node::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id)))
478 .exec(db)
479 .await
480 .map_err(map_sea_err)?;
481 Ok(())
482}
483
484#[instrument(
486 name = "cognee.db.relational.graph_storage.delete_edges_by_dataset",
487 level = "info",
488 skip_all,
489 fields(cognee.db.system = tracing::field::Empty),
490 err,
491)]
492pub async fn delete_edges_by_dataset(
493 db: &DatabaseConnection,
494 dataset_id: Uuid,
495) -> Result<(), DatabaseError> {
496 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
497 edge::Entity::delete_many()
498 .filter(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id)))
499 .exec(db)
500 .await
501 .map_err(map_sea_err)?;
502 Ok(())
503}
504
505#[instrument(
511 name = "cognee.db.relational.graph_storage.delete_nodes_for_data",
512 level = "info",
513 skip_all,
514 fields(cognee.db.system = tracing::field::Empty),
515 err,
516)]
517pub async fn delete_nodes_for_data(
518 db: &DatabaseConnection,
519 data_id: Uuid,
520 dataset_id: Uuid,
521) -> Result<(), DatabaseError> {
522 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
523 node::Entity::delete_many()
524 .filter(
525 Condition::all()
526 .add(node::Column::DataId.eq(uuid_hex::to_hex(data_id)))
527 .add(node::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
528 )
529 .exec(db)
530 .await
531 .map_err(map_sea_err)?;
532 Ok(())
533}
534
535#[instrument(
537 name = "cognee.db.relational.graph_storage.delete_edges_for_data",
538 level = "info",
539 skip_all,
540 fields(cognee.db.system = tracing::field::Empty),
541 err,
542)]
543pub async fn delete_edges_for_data(
544 db: &DatabaseConnection,
545 data_id: Uuid,
546 dataset_id: Uuid,
547) -> Result<(), DatabaseError> {
548 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
549 edge::Entity::delete_many()
550 .filter(
551 Condition::all()
552 .add(edge::Column::DataId.eq(uuid_hex::to_hex(data_id)))
553 .add(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
554 )
555 .exec(db)
556 .await
557 .map_err(map_sea_err)?;
558 Ok(())
559}
560
561#[instrument(
567 name = "cognee.db.relational.graph_storage.count_nodes_for_data",
568 level = "info",
569 skip_all,
570 fields(
571 cognee.db.system = tracing::field::Empty,
572 cognee.db.row_count = tracing::field::Empty,
573 ),
574 err,
575)]
576pub async fn count_nodes_for_data(
577 db: &DatabaseConnection,
578 data_id: Uuid,
579 dataset_id: Uuid,
580) -> Result<usize, DatabaseError> {
581 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
582 let count = node::Entity::find()
583 .filter(
584 Condition::all()
585 .add(node::Column::DataId.eq(uuid_hex::to_hex(data_id)))
586 .add(node::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
587 )
588 .count(db)
589 .await
590 .map_err(map_sea_err)?;
591 Span::current().record(COGNEE_DB_ROW_COUNT, count as i64);
592 Ok(count as usize)
593}
594
595#[instrument(
597 name = "cognee.db.relational.graph_storage.count_edges_for_data",
598 level = "info",
599 skip_all,
600 fields(
601 cognee.db.system = tracing::field::Empty,
602 cognee.db.row_count = tracing::field::Empty,
603 ),
604 err,
605)]
606pub async fn count_edges_for_data(
607 db: &DatabaseConnection,
608 data_id: Uuid,
609 dataset_id: Uuid,
610) -> Result<usize, DatabaseError> {
611 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
612 let count = edge::Entity::find()
613 .filter(
614 Condition::all()
615 .add(edge::Column::DataId.eq(uuid_hex::to_hex(data_id)))
616 .add(edge::Column::DatasetId.eq(uuid_hex::to_hex(dataset_id))),
617 )
618 .count(db)
619 .await
620 .map_err(map_sea_err)?;
621 Span::current().record(COGNEE_DB_ROW_COUNT, count as i64);
622 Ok(count as usize)
623}
624
625#[instrument(
634 name = "cognee.db.relational.graph_storage.get_unique_nodes_for_data",
635 level = "info",
636 skip_all,
637 fields(
638 cognee.db.system = tracing::field::Empty,
639 cognee.db.row_count = tracing::field::Empty,
640 ),
641 err,
642)]
643pub async fn get_unique_nodes_for_data(
644 db: &DatabaseConnection,
645 data_id: Uuid,
646 dataset_id: Uuid,
647) -> Result<Vec<GraphNode>, DatabaseError> {
648 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
649 let data_hex = uuid_hex::to_hex(data_id);
650 let dataset_hex = uuid_hex::to_hex(dataset_id);
651
652 let n2 = Alias::new("n2");
657 let shared = Query::select()
658 .expr(Expr::val(1))
659 .from_as(Alias::new("nodes"), n2.clone())
660 .and_where(Expr::col((n2.clone(), node::Column::DatasetId)).eq(dataset_hex.clone()))
661 .and_where(Expr::col((n2.clone(), node::Column::DataId)).ne(data_hex.clone()))
662 .and_where(
663 Expr::col((n2.clone(), node::Column::Slug))
664 .equals((Alias::new("nodes"), node::Column::Slug)),
665 )
666 .to_owned();
667
668 let rows: Vec<GraphNode> = node::Entity::find()
669 .filter(node::Column::DataId.eq(&data_hex))
670 .filter(node::Column::DatasetId.eq(&dataset_hex))
671 .filter(Expr::exists(shared).not())
672 .all(db)
673 .await
674 .map_err(map_sea_err)?
675 .into_iter()
676 .map(GraphNode::from)
677 .collect();
678 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
679 Ok(rows)
680}
681
682#[instrument(
685 name = "cognee.db.relational.graph_storage.get_unique_edges_for_data",
686 level = "info",
687 skip_all,
688 fields(
689 cognee.db.system = tracing::field::Empty,
690 cognee.db.row_count = tracing::field::Empty,
691 ),
692 err,
693)]
694pub async fn get_unique_edges_for_data(
695 db: &DatabaseConnection,
696 data_id: Uuid,
697 dataset_id: Uuid,
698) -> Result<Vec<GraphEdge>, DatabaseError> {
699 Span::current().record(COGNEE_DB_SYSTEM, database_system_label(db));
700 let data_hex = uuid_hex::to_hex(data_id);
701 let dataset_hex = uuid_hex::to_hex(dataset_id);
702
703 let e2 = Alias::new("e2");
707 let shared = Query::select()
708 .expr(Expr::val(1))
709 .from_as(Alias::new("edges"), e2.clone())
710 .and_where(Expr::col((e2.clone(), edge::Column::DatasetId)).eq(dataset_hex.clone()))
711 .and_where(Expr::col((e2.clone(), edge::Column::DataId)).ne(data_hex.clone()))
712 .and_where(
713 Expr::col((e2.clone(), edge::Column::Slug))
714 .equals((Alias::new("edges"), edge::Column::Slug)),
715 )
716 .to_owned();
717
718 let rows: Vec<GraphEdge> = edge::Entity::find()
719 .filter(edge::Column::DataId.eq(&data_hex))
720 .filter(edge::Column::DatasetId.eq(&dataset_hex))
721 .filter(Expr::exists(shared).not())
722 .all(db)
723 .await
724 .map_err(map_sea_err)?
725 .into_iter()
726 .map(GraphEdge::from)
727 .collect();
728 Span::current().record(COGNEE_DB_ROW_COUNT, rows.len() as i64);
729 Ok(rows)
730}