Skip to main content

fetchium_core/youtube/
intelligence.rs

1//! YouTube intelligence — cross-video fact checking, timelines, concepts, learning, teaching.
2
3use crate::youtube::types::*;
4use std::collections::HashMap;
5
6// ─── Cross-Video Fact Checking ─────────────────────────────────
7
8/// Cross-check facts across multiple videos using bigram-Jaccard claim matching.
9pub fn cross_check_facts(videos: &[VideoAnalysis], min_similarity: f64) -> Vec<FactCheckResult> {
10    let facts = extract_all_facts(videos);
11    let mut results = Vec::new();
12    let mut processed: std::collections::HashSet<String> = std::collections::HashSet::new();
13
14    for fact in &facts {
15        if processed.contains(&fact.claim) {
16            continue;
17        }
18
19        let mut supporting = Vec::new();
20        let mut contradicting = Vec::new();
21
22        for other in &facts {
23            if other.video_id == fact.video_id {
24                continue;
25            }
26
27            let sim = bigram_jaccard_similarity(&fact.claim, &other.claim);
28            if sim >= min_similarity {
29                if contains_negation_mismatch(&fact.claim, &other.claim) {
30                    contradicting.push(other.video_id.clone());
31                } else {
32                    supporting.push(other.video_id.clone());
33                }
34            }
35        }
36
37        let consensus = if !contradicting.is_empty() && supporting.is_empty() {
38            FactConsensus::Contradicted
39        } else if !contradicting.is_empty() {
40            FactConsensus::Disputed
41        } else if supporting.len() >= 2 {
42            FactConsensus::Confirmed
43        } else {
44            FactConsensus::Unverified
45        };
46
47        processed.insert(fact.claim.clone());
48        results.push(FactCheckResult {
49            claim: fact.claim.clone(),
50            supporting,
51            contradicting,
52            consensus,
53        });
54    }
55
56    results
57}
58
59/// Extract factual claims from all video transcripts.
60fn extract_all_facts(videos: &[VideoAnalysis]) -> Vec<VideoFact> {
61    let mut facts = Vec::new();
62
63    for video in videos {
64        if let Some(ref transcript) = video.transcript {
65            // Extract sentences that look like factual claims
66            for sentence in transcript.full_text.split(['.', '!', '?']) {
67                let trimmed = sentence.trim();
68                if trimmed.split_whitespace().count() >= 5 && is_factual_claim(trimmed) {
69                    facts.push(VideoFact {
70                        claim: trimmed.to_string(),
71                        video_id: video.metadata.video_id.clone(),
72                        timestamp_ms: None,
73                        confidence: 0.6,
74                    });
75                }
76            }
77        }
78    }
79
80    facts
81}
82
83/// Check if a sentence is likely a factual claim (not opinion/question).
84fn is_factual_claim(text: &str) -> bool {
85    let lower = text.to_lowercase();
86    // Skip opinions and hedged statements
87    let opinion_markers = [
88        "i think",
89        "i believe",
90        "in my opinion",
91        "personally",
92        "i feel",
93    ];
94    if opinion_markers.iter().any(|m| lower.contains(m)) {
95        return false;
96    }
97    // Skip questions
98    if lower.ends_with('?') {
99        return false;
100    }
101    // Look for factual markers
102    let fact_markers = [
103        "is",
104        "are",
105        "was",
106        "were",
107        "has",
108        "have",
109        "can",
110        "will",
111        "does",
112        "according to",
113    ];
114    fact_markers.iter().any(|m| lower.contains(m))
115}
116
117/// Check if two claims have a negation mismatch.
118fn contains_negation_mismatch(claim_a: &str, claim_b: &str) -> bool {
119    let a_lower = claim_a.to_lowercase();
120    let b_lower = claim_b.to_lowercase();
121
122    let a_negated = NEGATION_WORDS.iter().any(|n| a_lower.contains(n));
123    let b_negated = NEGATION_WORDS.iter().any(|n| b_lower.contains(n));
124
125    // If one is negated and the other isn't, it's a mismatch
126    a_negated != b_negated
127}
128
129/// Bigram Jaccard similarity.
130fn bigram_jaccard_similarity(a: &str, b: &str) -> f64 {
131    let a_bi = make_bigrams(&a.to_lowercase());
132    let b_bi = make_bigrams(&b.to_lowercase());
133
134    if a_bi.is_empty() && b_bi.is_empty() {
135        return 1.0;
136    }
137
138    let mut intersection = 0usize;
139    let mut union = 0usize;
140
141    let mut all_keys: std::collections::HashSet<&String> = std::collections::HashSet::new();
142    all_keys.extend(a_bi.keys());
143    all_keys.extend(b_bi.keys());
144
145    for key in all_keys {
146        let ca = a_bi.get(key).copied().unwrap_or(0);
147        let cb = b_bi.get(key).copied().unwrap_or(0);
148        intersection += ca.min(cb);
149        union += ca.max(cb);
150    }
151
152    if union == 0 {
153        0.0
154    } else {
155        intersection as f64 / union as f64
156    }
157}
158
159fn make_bigrams(s: &str) -> HashMap<String, usize> {
160    let chars: Vec<char> = s.chars().collect();
161    let mut bigrams = HashMap::new();
162    for window in chars.windows(2) {
163        let key = format!("{}{}", window[0], window[1]);
164        *bigrams.entry(key).or_insert(0) += 1;
165    }
166    bigrams
167}
168
169// ─── Topic Timeline ────────────────────────────────────────────
170
171/// Build a topic timeline from video publications.
172pub fn build_topic_timeline(videos: &[VideoAnalysis]) -> Vec<TimelineEntry> {
173    let mut entries: Vec<TimelineEntry> = videos
174        .iter()
175        .map(|v| {
176            let event = if let Some(ref transcript) = v.transcript {
177                // Extract first key point from transcript
178                transcript
179                    .key_moments
180                    .iter()
181                    .find(|m| {
182                        matches!(m.moment_type, MomentType::KeyPoint | MomentType::Conclusion)
183                    })
184                    .map(|m| m.text.clone())
185                    .unwrap_or_else(|| v.metadata.title.clone())
186            } else {
187                v.metadata.title.clone()
188            };
189
190            TimelineEntry {
191                date: v.metadata.published.clone(),
192                event,
193                video_id: v.metadata.video_id.clone(),
194                video_title: v.metadata.title.clone(),
195            }
196        })
197        .collect();
198
199    // Sort by date (approximate — using the published text)
200    entries.sort_by(|a, b| a.date.cmp(&b.date));
201    entries
202}
203
204// ─── Key Concept Extraction ────────────────────────────────────
205
206/// Extract key concepts from videos using definition pattern detection + TF-IDF.
207pub fn extract_key_concepts(videos: &[VideoAnalysis]) -> Vec<KeyConcept> {
208    let stopwords: std::collections::HashSet<&str> = STOPWORDS.iter().copied().collect();
209    let mut term_freq: HashMap<String, usize> = HashMap::new();
210    let mut term_videos: HashMap<String, Vec<String>> = HashMap::new();
211    let mut definitions: HashMap<String, String> = HashMap::new();
212
213    for video in videos {
214        if let Some(ref transcript) = video.transcript {
215            // Extract terms
216            let words: Vec<String> = transcript
217                .full_text
218                .to_lowercase()
219                .split_whitespace()
220                .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric()).to_string())
221                .filter(|w| w.len() > 3 && !stopwords.contains(w.as_str()))
222                .collect();
223
224            for word in &words {
225                *term_freq.entry(word.clone()).or_insert(0) += 1;
226                term_videos.entry(word.clone()).or_default();
227                if !term_videos[word].contains(&video.metadata.video_id) {
228                    term_videos
229                        .get_mut(word)
230                        .unwrap()
231                        .push(video.metadata.video_id.clone());
232                }
233            }
234
235            // Extract definitions
236            for pattern in DEFINITION_PATTERNS {
237                for sentence in transcript.full_text.split('.') {
238                    if let Some(pos) = sentence.to_lowercase().find(pattern) {
239                        let term = sentence[..pos]
240                            .split_whitespace()
241                            .last()
242                            .unwrap_or("")
243                            .to_lowercase();
244                        if term.len() > 2 && !stopwords.contains(term.as_str()) {
245                            definitions
246                                .entry(term)
247                                .or_insert_with(|| sentence.trim().to_string());
248                        }
249                    }
250                }
251            }
252        }
253    }
254
255    // Sort by frequency, take top 20
256    let mut sorted: Vec<(String, usize)> = term_freq.into_iter().collect();
257    sorted.sort_by(|a, b| b.1.cmp(&a.1));
258
259    sorted
260        .into_iter()
261        .take(20)
262        .map(|(term, freq)| KeyConcept {
263            definition: definitions.get(&term).cloned(),
264            source_videos: term_videos.get(&term).cloned().unwrap_or_default(),
265            term,
266            frequency: freq,
267        })
268        .collect()
269}
270
271// ─── Learning Path ─────────────────────────────────────────────
272
273/// Generate a learning path from videos ordered by difficulty.
274pub fn generate_learning_path(videos: &[VideoAnalysis]) -> Vec<LearningStep> {
275    let mut steps: Vec<LearningStep> = videos
276        .iter()
277        .map(|v| {
278            let difficulty = estimate_difficulty(v);
279            let concepts = extract_video_concepts(v);
280
281            LearningStep {
282                order: 0, // will be set after sorting
283                video_id: v.metadata.video_id.clone(),
284                title: v.metadata.title.clone(),
285                difficulty,
286                key_concepts: concepts,
287                estimated_minutes: v.metadata.duration_secs / 60,
288            }
289        })
290        .collect();
291
292    // Sort by difficulty level
293    steps.sort_by_key(|a| a.difficulty);
294
295    // Assign order
296    for (i, step) in steps.iter_mut().enumerate() {
297        step.order = i + 1;
298    }
299
300    steps
301}
302
303/// Estimate video difficulty based on word complexity and content.
304fn estimate_difficulty(video: &VideoAnalysis) -> DifficultyLevel {
305    let mut complexity_score = 0.0f64;
306
307    // Title complexity
308    let title_words = video.metadata.title.split_whitespace().count();
309    if title_words > 10 {
310        complexity_score += 0.2;
311    }
312
313    // Transcript word complexity
314    if let Some(ref transcript) = video.transcript {
315        let avg_word_len = if transcript.word_count > 0 {
316            transcript.full_text.len() as f64 / transcript.word_count as f64
317        } else {
318            4.0
319        };
320        // Higher avg word length = more complex
321        complexity_score += (avg_word_len - 4.0).max(0.0) * 0.1;
322
323        // More key moments = more structured = potentially more advanced
324        complexity_score += (transcript.key_moments.len() as f64 * 0.02).min(0.3);
325    }
326
327    // Duration (longer = often more advanced)
328    let dur_min = video.metadata.duration_secs as f64 / 60.0;
329    if dur_min > 30.0 {
330        complexity_score += 0.2;
331    } else if dur_min > 15.0 {
332        complexity_score += 0.1;
333    }
334
335    // Keywords indicating difficulty
336    let lower_title = video.metadata.title.to_lowercase();
337    let beginner_words = [
338        "beginner",
339        "introduction",
340        "basics",
341        "101",
342        "getting started",
343        "first",
344    ];
345    let advanced_words = [
346        "advanced",
347        "deep dive",
348        "internals",
349        "architecture",
350        "expert",
351        "master",
352    ];
353
354    if beginner_words.iter().any(|w| lower_title.contains(w)) {
355        complexity_score -= 0.3;
356    }
357    if advanced_words.iter().any(|w| lower_title.contains(w)) {
358        complexity_score += 0.3;
359    }
360
361    match complexity_score {
362        s if s < 0.2 => DifficultyLevel::Beginner,
363        s if s < 0.5 => DifficultyLevel::Intermediate,
364        s if s < 0.8 => DifficultyLevel::Advanced,
365        _ => DifficultyLevel::Expert,
366    }
367}
368
369/// Extract main concepts from a single video.
370fn extract_video_concepts(video: &VideoAnalysis) -> Vec<String> {
371    let stopwords: std::collections::HashSet<&str> = STOPWORDS.iter().copied().collect();
372    let mut freq: HashMap<String, usize> = HashMap::new();
373
374    // From title
375    for word in video.metadata.title.to_lowercase().split_whitespace() {
376        let clean = word
377            .trim_matches(|c: char| !c.is_alphanumeric())
378            .to_string();
379        if clean.len() > 3 && !stopwords.contains(clean.as_str()) {
380            *freq.entry(clean).or_insert(0) += 3;
381        }
382    }
383
384    // From keywords
385    for kw in &video.metadata.keywords {
386        let clean = kw.to_lowercase();
387        if !stopwords.contains(clean.as_str()) {
388            *freq.entry(clean).or_insert(0) += 2;
389        }
390    }
391
392    let mut sorted: Vec<(String, usize)> = freq.into_iter().collect();
393    sorted.sort_by(|a, b| b.1.cmp(&a.1));
394    sorted.into_iter().take(5).map(|(t, _)| t).collect()
395}
396
397// ─── Teaching Mode ─────────────────────────────────────────────
398
399/// Generate teaching content (flashcards + quiz) from video analyses.
400pub fn generate_teaching(videos: &[VideoAnalysis]) -> TeachingContent {
401    let concepts = extract_key_concepts(videos);
402    let summary = generate_summary(videos);
403    let flashcards = generate_flashcards(&concepts);
404    let quiz_questions = generate_quiz(&concepts, videos);
405
406    TeachingContent {
407        summary,
408        flashcards,
409        quiz_questions,
410    }
411}
412
413/// Generate a summary from video analyses.
414fn generate_summary(videos: &[VideoAnalysis]) -> String {
415    let mut parts = Vec::new();
416    for video in videos {
417        if let Some(ref transcript) = video.transcript {
418            // Take first key point or first 200 chars of transcript
419            let excerpt = transcript
420                .key_moments
421                .iter()
422                .find(|m| matches!(m.moment_type, MomentType::KeyPoint | MomentType::Conclusion))
423                .map(|m| m.text.clone())
424                .unwrap_or_else(|| transcript.full_text.chars().take(200).collect::<String>());
425            parts.push(format!("**{}**: {}", video.metadata.title, excerpt));
426        } else {
427            parts.push(format!(
428                "**{}**: {}",
429                video.metadata.title,
430                video
431                    .metadata
432                    .description
433                    .chars()
434                    .take(100)
435                    .collect::<String>()
436            ));
437        }
438    }
439    parts.join("\n\n")
440}
441
442/// Generate flashcards from key concepts.
443fn generate_flashcards(concepts: &[KeyConcept]) -> Vec<Flashcard> {
444    concepts
445        .iter()
446        .filter(|c| c.definition.is_some())
447        .take(10)
448        .map(|c| Flashcard {
449            front: format!("What is {}?", c.term),
450            back: c.definition.clone().unwrap_or_default(),
451        })
452        .collect()
453}
454
455/// Generate quiz questions from concepts and video content.
456fn generate_quiz(concepts: &[KeyConcept], _videos: &[VideoAnalysis]) -> Vec<QuizQuestion> {
457    concepts
458        .iter()
459        .filter(|c| c.definition.is_some() && c.frequency > 2)
460        .take(5)
461        .map(|concept| {
462            let def = concept.definition.clone().unwrap_or_default();
463            QuizQuestion {
464                question: format!("Which of the following best describes '{}'?", concept.term),
465                options: vec![
466                    def.clone(),
467                    format!("A type of data structure used in {}", concept.term),
468                    format!("An algorithm for processing {}", concept.term),
469                    "None of the above".to_string(),
470                ],
471                correct_index: 0,
472                explanation: format!("The correct definition is: {def}"),
473            }
474        })
475        .collect()
476}
477
478/// Analyze viral content patterns from videos.
479pub fn analyze_viral_patterns(videos: &[VideoAnalysis]) -> Vec<ContentIdea> {
480    let mut ideas = Vec::new();
481
482    // Find common themes in high-engagement videos
483    let high_engagement: Vec<&VideoAnalysis> = videos
484        .iter()
485        .filter(|v| {
486            let like_ratio = if v.metadata.view_count > 0 {
487                v.metadata.like_count as f64 / v.metadata.view_count as f64
488            } else {
489                0.0
490            };
491            like_ratio > 0.03 || v.metadata.view_count > 100_000
492        })
493        .collect();
494
495    if high_engagement.is_empty() {
496        return ideas;
497    }
498
499    // Extract common keywords from viral videos
500    let stopwords: std::collections::HashSet<&str> = STOPWORDS.iter().copied().collect();
501    let mut keyword_freq: HashMap<String, usize> = HashMap::new();
502    for video in &high_engagement {
503        for kw in &video.metadata.keywords {
504            let lower = kw.to_lowercase();
505            if !stopwords.contains(lower.as_str()) {
506                *keyword_freq.entry(lower).or_insert(0) += 1;
507            }
508        }
509        // Title words
510        for word in video.metadata.title.to_lowercase().split_whitespace() {
511            let clean = word
512                .trim_matches(|c: char| !c.is_alphanumeric())
513                .to_string();
514            if clean.len() > 3 && !stopwords.contains(clean.as_str()) {
515                *keyword_freq.entry(clean).or_insert(0) += 1;
516            }
517        }
518    }
519
520    let mut sorted: Vec<(String, usize)> =
521        keyword_freq.into_iter().filter(|(_, v)| *v >= 2).collect();
522    sorted.sort_by(|a, b| b.1.cmp(&a.1));
523
524    for (topic, freq) in sorted.into_iter().take(5) {
525        // Analyze duration sweet spot
526        let avg_duration: u64 = high_engagement
527            .iter()
528            .map(|v| v.metadata.duration_secs)
529            .sum::<u64>()
530            / high_engagement.len().max(1) as u64;
531
532        ideas.push(ContentIdea {
533            topic: topic.clone(),
534            angle: format!(
535                "Trending topic '{}' appears in {} viral videos",
536                topic, freq
537            ),
538            suggested_duration_secs: avg_duration,
539            keyword_density: freq,
540            reference_videos: high_engagement
541                .iter()
542                .filter(|v| {
543                    v.metadata.title.to_lowercase().contains(&topic)
544                        || v.metadata
545                            .keywords
546                            .iter()
547                            .any(|k| k.to_lowercase() == topic)
548                })
549                .map(|v| v.metadata.video_id.clone())
550                .collect(),
551        });
552    }
553
554    ideas
555}
556
557/// Content idea generated from viral analysis.
558#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
559pub struct ContentIdea {
560    pub topic: String,
561    pub angle: String,
562    pub suggested_duration_secs: u64,
563    pub keyword_density: usize,
564    pub reference_videos: Vec<String>,
565}
566
567#[cfg(test)]
568mod tests {
569    use super::*;
570    use crate::youtube::metadata::score_channel_credibility;
571
572    fn make_video(id: &str, title: &str, transcript_text: Option<&str>) -> VideoAnalysis {
573        let channel = ChannelInfo {
574            name: "TestCh".into(),
575            id: "UC1".into(),
576            subscriber_count: Some(10_000),
577            verified: false,
578        };
579        VideoAnalysis {
580            metadata: VideoMetadata {
581                video_id: id.into(),
582                title: title.into(),
583                description: "Test description".into(),
584                channel: channel.clone(),
585                duration_secs: 600,
586                view_count: 10_000,
587                like_count: 500,
588                published: "2 weeks ago".into(),
589                keywords: vec!["rust".into(), "programming".into()],
590                chapters: vec![],
591                links: vec![],
592                thumbnail_url: None,
593                is_live: false,
594            },
595            transcript: transcript_text.map(|text| EnhancedTranscript {
596                video_id: id.into(),
597                language: "en".into(),
598                entries: vec![],
599                speakers: vec![],
600                key_moments: vec![],
601                full_text: text.into(),
602                word_count: text.split_whitespace().count(),
603                source: TranscriptSource::YouTubeTimedtext,
604                quality_score: 1.0,
605            }),
606            comments: None,
607            credibility: score_channel_credibility(&channel),
608        }
609    }
610
611    #[test]
612    fn cross_check_basic() {
613        let videos = vec![
614            make_video(
615                "v1",
616                "Rust is fast",
617                Some("Rust is a systems programming language that is very fast and memory safe"),
618            ),
619            make_video(
620                "v2",
621                "Rust performance",
622                Some("Rust is a systems programming language known for being fast and safe"),
623            ),
624            make_video(
625                "v3",
626                "Python vs Rust",
627                Some("Python is not a systems programming language but Rust is"),
628            ),
629        ];
630        let results = cross_check_facts(&videos, 0.4);
631        assert!(!results.is_empty());
632    }
633
634    #[test]
635    fn is_factual_claim_check() {
636        assert!(is_factual_claim("Rust is a systems programming language"));
637        assert!(!is_factual_claim("I think Rust is great"));
638        assert!(!is_factual_claim("Is Rust good?"));
639    }
640
641    #[test]
642    fn negation_mismatch_detection() {
643        assert!(contains_negation_mismatch(
644            "Rust is memory safe",
645            "Rust is not memory safe"
646        ));
647        assert!(!contains_negation_mismatch(
648            "Rust is fast",
649            "Rust is very fast"
650        ));
651    }
652
653    #[test]
654    fn topic_timeline_basic() {
655        let videos = vec![
656            make_video("v1", "Early Topic", None),
657            make_video("v2", "Later Topic", None),
658        ];
659        let timeline = build_topic_timeline(&videos);
660        assert_eq!(timeline.len(), 2);
661    }
662
663    #[test]
664    fn key_concepts_extraction() {
665        let videos = vec![
666            make_video("v1", "Rust Tutorial", Some("async programming in rust allows you to write concurrent code async programming is powerful async runtime tokio")),
667        ];
668        let concepts = extract_key_concepts(&videos);
669        assert!(!concepts.is_empty());
670        assert!(concepts
671            .iter()
672            .any(|c| c.term.contains("async") || c.term.contains("rust")));
673    }
674
675    #[test]
676    fn learning_path_ordering() {
677        let videos = vec![
678            make_video(
679                "v1",
680                "Advanced Rust Internals Deep Dive",
681                Some("Complex topic about ownership and borrowing"),
682            ),
683            make_video(
684                "v2",
685                "Rust for Beginners - Getting Started",
686                Some("Hello world in rust"),
687            ),
688        ];
689        let path = generate_learning_path(&videos);
690        assert_eq!(path.len(), 2);
691        assert!(path[0].difficulty <= path[1].difficulty);
692    }
693
694    #[test]
695    fn teaching_generation() {
696        let videos = vec![make_video(
697            "v1",
698            "Rust Closures",
699            Some("A closure is defined as a function that captures variables from its environment"),
700        )];
701        let teaching = generate_teaching(&videos);
702        assert!(!teaching.summary.is_empty());
703    }
704
705    #[test]
706    fn difficulty_estimation() {
707        let beginner = make_video("v1", "Introduction to Rust - Getting Started Basics", None);
708        let advanced = make_video("v2", "Advanced Rust Architecture Deep Dive Internals", None);
709        assert!(estimate_difficulty(&beginner) < estimate_difficulty(&advanced));
710    }
711
712    #[test]
713    fn viral_patterns_empty() {
714        let videos: Vec<VideoAnalysis> = vec![];
715        let ideas = analyze_viral_patterns(&videos);
716        assert!(ideas.is_empty());
717    }
718}