tokensave 7.12.1

Code intelligence tool that builds a semantic knowledge graph from Rust, Go, Java, Scala, TypeScript, Python, C, C++, Kotlin, C#, Swift, and many more codebases
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168

//! HTML and CSS extraction — #507.
//!
//! Both file types were skipped outright ("no registered extractor"), so a
//! project's templates and stylesheets were invisible to every graph query.
//! Neither language has callable symbols, so what these extractors record is
//! the names a file *defines* and the files it pulls in.

#![cfg(all(feature = "lang-html", feature = "lang-css"))]

use tokensave::extraction::{CssExtractor, HtmlExtractor, LanguageExtractor};
use tokensave::types::{EdgeKind, NodeKind};

const PAGE: &str = r#"<!DOCTYPE html>
<html>
  <head>
    <link rel="stylesheet" href="styles/site.css">
    <link rel="icon" href="favicon.ico">
    <script src="app.js"></script>
  </head>
  <body>
    <div id="app" class="container">
      <my-widget label="hello"></my-widget>
      <div class="row">plain</div>
    </div>
    <template id="row-template"></template>
  </body>
</html>
"#;

const SHEET: &str = r#"@import "reset.css";

:root {
  --brand: #336699;
  --spacing: 8px;
}

.container {
  color: var(--brand);
  padding: var(--spacing);
}

#app {
  margin: 0;
}

.container, .row {
  display: flex;
}

@keyframes fade-in {
  from { opacity: 0; }
  to { opacity: 1; }
}

@media (min-width: 40rem) {
  .row {
    gap: var(--spacing);
  }
}
"#;

fn names_of(result: &tokensave::types::ExtractionResult, kind: NodeKind) -> Vec<String> {
    let mut names: Vec<String> = result
        .nodes
        .iter()
        .filter(|n| n.kind == kind)
        .map(|n| n.name.clone())
        .collect();
    names.sort();
    names
}

#[test]
fn html_records_ids_components_and_imports() {
    let result = HtmlExtractor.extract("index.html", PAGE);
    assert!(result.errors.is_empty(), "errors: {:?}", result.errors);

    // Elements carrying an `id`, whatever the tag.
    assert_eq!(names_of(&result, NodeKind::Field), ["app", "row-template"]);

    // A custom element names a component; builtin tags do not.
    assert_eq!(names_of(&result, NodeKind::Class), ["my-widget"]);

    // The page's imports: a stylesheet link and a script source. `rel="icon"`
    // is a link but not an import of anything the graph can follow.
    assert_eq!(
        names_of(&result, NodeKind::Use),
        ["app.js", "styles/site.css"]
    );

    // Every emitted node hangs off the file.
    let contains = result
        .edges
        .iter()
        .filter(|e| e.kind == EdgeKind::Contains)
        .count();
    assert_eq!(
        contains,
        result.nodes.len() - 1,
        "one edge per non-file node"
    );
}

#[test]
fn html_emits_no_reference_for_a_class_attribute() {
    // Class names are ordinary words. Resolving them by bare name is how a
    // stylesheet ends up owning an edge into unrelated code (#503).
    let result = HtmlExtractor.extract("index.html", PAGE);
    assert!(
        result.unresolved_refs.is_empty(),
        "markup should not guess at cross-file names: {:?}",
        result.unresolved_refs
    );
}

#[test]
fn css_records_selectors_properties_and_keyframes() {
    let result = CssExtractor.extract("site.css", SHEET);
    assert!(result.errors.is_empty(), "errors: {:?}", result.errors);

    // A selector repeated under a media query is one node, not three.
    assert_eq!(names_of(&result, NodeKind::Class), ["container", "row"]);
    assert_eq!(names_of(&result, NodeKind::Field), ["app"]);
    assert_eq!(names_of(&result, NodeKind::Const), ["--brand", "--spacing"]);
    assert_eq!(names_of(&result, NodeKind::Module), ["fade-in"]);
    assert_eq!(names_of(&result, NodeKind::Use), ["reset.css"]);
}

#[test]
fn css_references_custom_properties_and_nothing_else() {
    let result = CssExtractor.extract("site.css", SHEET);
    let mut referenced: Vec<String> = result
        .unresolved_refs
        .iter()
        .map(|r| r.reference_name.clone())
        .collect();
    referenced.sort();
    referenced.dedup();

    // `var(--brand)` and `var(--spacing)` are followed; `#336699`, `0` and
    // every builtin function are not.
    assert_eq!(referenced, ["--brand", "--spacing"]);
    assert!(
        result
            .unresolved_refs
            .iter()
            .all(|r| r.reference_name.starts_with("--")),
        "only custom properties are safe to resolve by bare name"
    );
}

#[test]
fn both_extractors_claim_their_extensions() {
    assert!(HtmlExtractor.extensions().contains(&"html"));
    assert!(HtmlExtractor.extensions().contains(&"htm"));
    assert!(CssExtractor.extensions().contains(&"css"));
}

#[test]
fn malformed_input_does_not_panic_or_error() {
    let broken_html = "<div id=\"a\"><span class=</div><my-thing>";
    let broken_css = "@import ; .a { color: var(--x) } #{}";
    assert!(HtmlExtractor
        .extract("b.html", broken_html)
        .errors
        .is_empty());
    assert!(CssExtractor.extract("b.css", broken_css).errors.is_empty());
}