1use once_cell::sync::Lazy;
2use rustc_hash::FxHashSet;
3
4use crate::config::extensions::{CODE_EXTENSIONS, DOC_EXTENSIONS};
5
6fn set_from(words: &[&str]) -> FxHashSet<String> {
7 words.iter().map(|w| w.to_string()).collect()
8}
9
10static PY_KEYWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
11 set_from(&[
12 "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del",
13 "elif", "else", "except", "false", "finally", "for", "from", "global", "if", "import",
14 "in", "is", "lambda", "none", "nonlocal", "not", "or", "pass", "raise", "return", "true",
15 "try", "while", "with", "yield",
16 ])
17});
18
19static CODE_STOPWORDS_BASE: Lazy<FxHashSet<String>> = Lazy::new(|| {
20 set_from(&[
21 "todo",
22 "fixme",
23 "note",
24 "hack",
25 "xxx",
26 "foo",
27 "bar",
28 "baz",
29 "qux",
30 "tmp",
31 "temp",
32 "self",
33 "cls",
34 "get",
35 "set",
36 "put",
37 "delete",
38 "remove",
39 "update",
40 "create",
41 "read",
42 "write",
43 "open",
44 "close",
45 "start",
46 "stop",
47 "run",
48 "execute",
49 "call",
50 "invoke",
51 "handle",
52 "process",
53 "validate",
54 "check",
55 "test",
56 "init",
57 "setup",
58 "teardown",
59 "load",
60 "save",
61 "parse",
62 "format",
63 "convert",
64 "transform",
65 "render",
66 "build",
67 "make",
68 "copy",
69 "clone",
70 "reset",
71 "clear",
72 "flush",
73 "send",
74 "receive",
75 "emit",
76 "insert",
77 "append",
78 "extend",
79 "split",
80 "join",
81 "trim",
82 "strip",
83 "contains",
84 "includes",
85 "push",
86 "pop",
87 "concat",
88 "slice",
89 "each",
90 "print",
91 "println",
92 "printf",
93 "sprintf",
94 "fprintf",
95 "abort",
96 "retry",
97 "apply",
98 "resolve",
99 "reject",
100 "defer",
101 "sleep",
102 "wait",
103 "poll",
104 "encode",
105 "decode",
106 "encrypt",
107 "decrypt",
108 "hash",
109 "sign",
110 "verify",
111 "register",
112 "unregister",
113 "mount",
114 "unmount",
115 "enable",
116 "disable",
117 "show",
118 "hide",
119 "lock",
120 "unlock",
121 "acquire",
122 "release",
123 "commit",
124 "rollback",
125 "begin",
126 "end",
127 "value",
128 "values",
129 "result",
130 "results",
131 "item",
132 "items",
133 "element",
134 "elements",
135 "key",
136 "keys",
137 "text",
138 "num",
139 "number",
140 "count",
141 "index",
142 "idx",
143 "size",
144 "length",
145 "len",
146 "array",
147 "dict",
148 "obj",
149 "object",
150 "args",
151 "kwargs",
152 "params",
153 "options",
154 "err",
155 "error",
156 "errors",
157 "msg",
158 "message",
159 "messages",
160 "log",
161 "logger",
162 "debug",
163 "info",
164 "warn",
165 "warning",
166 "func",
167 "function",
168 "method",
169 "callback",
170 "handler",
171 "listener",
172 "event",
173 "events",
174 "request",
175 "response",
176 "req",
177 "res",
178 "body",
179 "header",
180 "headers",
181 "query",
182 "param",
183 "id",
184 "uid",
185 "uuid",
186 "var",
187 "let",
188 "const",
189 "new",
190 "this",
191 "null",
192 "nil",
193 "none",
194 "void",
195 "true",
196 "false",
197 "static",
198 "final",
199 "abstract",
200 "virtual",
201 "override",
202 "public",
203 "private",
204 "protected",
205 "internal",
206 "extern",
207 "inline",
208 "volatile",
209 "mutable",
210 "readonly",
211 "super",
212 "extends",
213 "implements",
214 "import",
215 "export",
216 "require",
217 "include",
218 "using",
219 "typedef",
220 "define",
221 "ifdef",
222 "ifndef",
223 "endif",
224 "elif",
225 "else",
226 "case",
227 "switch",
228 "break",
229 "continue",
230 "return",
231 "goto",
232 "sizeof",
233 "typeof",
234 "instanceof",
235 "throw",
236 "throws",
237 "try",
238 "catch",
239 "finally",
240 "raise",
241 "except",
242 "yield",
243 "lambda",
244 "proc",
245 "sub",
246 "def",
247 "async",
248 "await",
249 "int",
250 "uint",
251 "long",
252 "float",
253 "double",
254 "char",
255 "byte",
256 "short",
257 "signed",
258 "unsigned",
259 "bool",
260 "boolean",
261 "string",
262 "str",
263 "size_t",
264 "usize",
265 "isize",
266 "int8",
267 "int16",
268 "int32",
269 "int64",
270 "uint8",
271 "uint16",
272 "uint32",
273 "uint64",
274 "float32",
275 "float64",
276 "__init__",
277 "__main__",
278 "__name__",
279 "__str__",
280 "__repr__",
281 "__eq__",
282 "__hash__",
283 "__len__",
284 "__iter__",
285 "__next__",
286 "__enter__",
287 "__exit__",
288 "echo",
289 "done",
290 "local",
291 "eval",
292 "exec",
293 "exit",
294 "trap",
295 "bash",
296 "then",
297 "esac",
298 "fi",
299 "do",
300 "for",
301 "while",
302 "until",
303 "shift",
304 "unset",
305 "declare",
306 "alias",
307 "html",
308 "head",
309 "div",
310 "span",
311 "form",
312 "button",
313 "img",
314 "href",
315 "style",
316 "display",
317 "margin",
318 "padding",
319 "border",
320 "color",
321 "width",
322 "height",
323 "flex",
324 "grid",
325 "onclick",
326 "onchange",
327 "onsubmit",
328 "classname",
329 ])
330});
331
332static DOMAIN_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
333 set_from(&[
334 "add",
335 "sort",
336 "filter",
337 "find",
338 "select",
339 "merge",
340 "match",
341 "replace",
342 "dispatch",
343 "notify",
344 "subscribe",
345 "publish",
346 "connect",
347 "disconnect",
348 "bind",
349 "listen",
350 "accept",
351 "every",
352 "some",
353 "reduce",
354 "splice",
355 "unshift",
356 "throw",
357 "catch",
358 "create",
359 "destroy",
360 "dispose",
361 "finalize",
362 "data",
363 "name",
364 "names",
365 "list",
366 "map",
367 "config",
368 "settings",
369 "context",
370 "ctx",
371 "state",
372 "status",
373 "type",
374 "kind",
375 "mode",
376 "flag",
377 "flags",
378 "path",
379 "file",
380 "dir",
381 "url",
382 "uri",
383 "host",
384 "port",
385 "input",
386 "output",
387 "source",
388 "target",
389 "dest",
390 "src",
391 "dst",
392 "user",
393 "users",
394 "model",
395 "models",
396 "view",
397 "views",
398 "service",
399 "services",
400 "client",
401 "server",
402 "api",
403 "app",
404 "main",
405 "util",
406 "utils",
407 "helper",
408 "helpers",
409 "common",
410 "base",
411 "core",
412 "default",
413 "defaults",
414 "version",
415 "label",
416 "labels",
417 "tag",
418 "tags",
419 "level",
420 "scope",
421 "token",
422 "tokens",
423 "task",
424 "tasks",
425 "job",
426 "jobs",
427 "step",
428 "steps",
429 "stage",
430 "cache",
431 "timeout",
432 "interval",
433 "duration",
434 "timestamp",
435 "channel",
436 "buffer",
437 "queue",
438 "stack",
439 "heap",
440 "node",
441 "nodes",
442 "edge",
443 "edges",
444 "child",
445 "children",
446 "parent",
447 "root",
448 "leaf",
449 "link",
450 "ref",
451 "reference",
452 "instance",
453 "schema",
454 "table",
455 "column",
456 "row",
457 "field",
458 "record",
459 "entry",
460 "spec",
461 "env",
462 "namespace",
463 "prefix",
464 "suffix",
465 "pattern",
466 "template",
467 "factory",
468 "builder",
469 "adapter",
470 "proxy",
471 "wrapper",
472 "manager",
473 "controller",
474 "provider",
475 "consumer",
476 "producer",
477 "worker",
478 "pool",
479 "connection",
480 "session",
481 "stream",
482 "pipe",
483 "socket",
484 "signal",
485 "trigger",
486 "action",
487 "command",
488 "rule",
489 "rules",
490 "policy",
491 "strategy",
492 "middleware",
493 "plugin",
494 "module",
495 "package",
496 "component",
497 "container",
498 "registry",
499 "repository",
500 "interface",
501 "struct",
502 "enum",
503 "union",
504 "trait",
505 "impl",
506 "class",
507 "implement",
508 "inherit",
509 "mixin",
510 "tostring",
511 "hashcode",
512 "equals",
513 "usestate",
514 "useeffect",
515 "usecontext",
516 "usereducer",
517 "usecallback",
518 "usememo",
519 "useref",
520 "uselayouteffect",
521 "useimperativehandle",
522 "usedebugvalue",
523 "useid",
524 "usetransition",
525 "usedeferredvalue",
526 "createcontext",
527 "forwardref",
528 "createref",
529 "suspense",
530 "strictmode",
531 "profiler",
532 "usenavigate",
533 "useparams",
534 "uselocation",
535 "usesearchparams",
536 "useloaderdata",
537 "useactiondata",
538 "usefetcher",
539 "useoutletcontext",
540 "usedispatch",
541 "useselector",
542 "usestore",
543 "usequery",
544 "usemutation",
545 "usesubscription",
546 "definecomponent",
547 "defineprops",
548 "defineemits",
549 "defineslots",
550 "definemodel",
551 "defineexpose",
552 "toref",
553 "torefs",
554 "reactive",
555 "computed",
556 "onmounted",
557 "onunmounted",
558 "onbeforemount",
559 "onupdated",
560 "watcheffect",
561 "nexttick",
562 "ngoninit",
563 "ngondestroy",
564 "ngonchanges",
565 "ngafterviewinit",
566 "console",
567 "document",
568 "window",
569 "navigator",
570 "location",
571 "history",
572 "fetch",
573 "promise",
574 "undefined",
575 "prototype",
576 "constructor",
577 "exports",
578 "props",
579 "effect",
580 "memo",
581 "styles",
582 "innerhtml",
583 "textcontent",
584 "appendchild",
585 "createelement",
586 "queryselector",
587 "getelementbyid",
588 "addeventlistener",
589 "resource",
590 "variable",
591 "locals",
592 "terraform",
593 "jsonencode",
594 "cidr",
595 "subnet",
596 "ingress",
597 "egress",
598 "protocol",
599 "cidr_blocks",
600 "security_group",
601 "arn",
602 "vpc",
603 "aws",
604 "gcp",
605 "azure",
606 "region",
607 "zone",
608 "cluster",
609 "replicas",
610 "selector",
611 "metadata",
612 "annotations",
613 "volumes",
614 "ports",
615 "image",
616 "containers",
617 "resources",
618 "limits",
619 "requests",
620 "apiversion",
621 "configmap",
622 "secret",
623 "deployment",
624 "statefulset",
625 "daemonset",
626 "cronjob",
627 "serviceaccount",
628 "role",
629 "rolebinding",
630 "clusterrole",
631 "install",
632 "deploy",
633 "pipeline",
634 "script",
635 "pull",
636 "uses",
637 "with",
638 "needs",
639 "runs",
640 "checkout",
641 "artifact",
642 "artifacts",
643 "workflow",
644 "schedule",
645 "cron",
646 "branch",
647 "branches",
648 "release",
649 "from",
650 "workdir",
651 "expose",
652 "entrypoint",
653 "volume",
654 "arg",
655 "healthcheck",
656 "stopsignal",
657 "describe",
658 "it",
659 "expect",
660 "assert",
661 "should",
662 "mock",
663 "stub",
664 "spy",
665 "before",
666 "after",
667 "beforeall",
668 "afterall",
669 "beforeeach",
670 "aftereach",
671 "suite",
672 "fixture",
673 "given",
674 "when",
675 "insert",
676 "update",
677 "delete",
678 "from",
679 "where",
680 "join",
681 "inner",
682 "outer",
683 "left",
684 "right",
685 "group",
686 "order",
687 "limit",
688 "offset",
689 "having",
690 "distinct",
691 "count",
692 "sum",
693 "avg",
694 "min",
695 "max",
696 "constraint",
697 "primary",
698 "foreign",
699 "unique",
700 "not",
701 "and",
702 "or",
703 "between",
704 "exists",
705 "values",
706 "into",
707 "alter",
708 "drop",
709 "truncate",
710 "grant",
711 "revoke",
712 "body",
713 "title",
714 "meta",
715 "script",
716 "label",
717 "select",
718 "option",
719 "thead",
720 "tbody",
721 "class",
722 "type",
723 "name",
724 "value",
725 "placeholder",
726 "required",
727 "disabled",
728 "checked",
729 "selected",
730 "hidden",
731 "http",
732 "https",
733 "tcp",
734 "udp",
735 "dns",
736 "ssl",
737 "tls",
738 "cert",
739 "certificate",
740 "content",
741 "authorization",
742 "bearer",
743 "basic",
744 "digest",
745 "origin",
746 "referer",
747 "cookie",
748 "cookies",
749 "redirect",
750 "gateway",
751 "load",
752 "balance",
753 "upstream",
754 "downstream",
755 "endpoint",
756 "route",
757 "routes",
758 "router",
759 "method",
760 "code",
761 "java",
762 "python",
763 "golang",
764 "ruby",
765 "rust",
766 "swift",
767 "kotlin",
768 "scala",
769 "perl",
770 "elixir",
771 "haskell",
772 "clojure",
773 "erlang",
774 "typescript",
775 "javascript",
776 "csharp",
777 "cplusplus",
778 "alpine",
779 "ubuntu",
780 "debian",
781 "linux",
782 "darwin",
783 "windows",
784 "time",
785 "math",
786 "system",
787 "vector",
788 "hashmap",
789 "hashset",
790 "treemap",
791 "arraylist",
792 "linkedlist",
793 "optional",
794 "future",
795 "pair",
796 "tuple",
797 "collection",
798 "collections",
799 "iterator",
800 "iterable",
801 "comparable",
802 "serializable",
803 "cloneable",
804 "runnable",
805 "callable",
806 "supplier",
807 "predicate",
808 "comparator",
809 "json",
810 "xml",
811 "yaml",
812 "toml",
813 "csv",
814 "email",
815 "phone",
816 "address",
817 "date",
818 "datetime",
819 "password",
820 "username",
821 "auth",
822 "login",
823 "logout",
824 "admin",
825 "account",
826 "profile",
827 "permission",
828 "team",
829 "organization",
830 "project",
831 "description",
832 "category",
833 "comment",
834 "comments",
835 "like",
836 "share",
837 "search",
838 "page",
839 "pages",
840 "upload",
841 "download",
842 "notification",
843 "notifications",
844 "alert",
845 "alerts",
846 "validation",
847 "production",
848 "staging",
849 "development",
850 "testing",
851 "localhost",
852 "unknown",
853 "refs",
854 "heads",
855 "remote",
856 "master",
857 "rebase",
858 "cherry",
859 "stash",
860 ])
861});
862
863pub static CODE_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
864 let mut combined = CODE_STOPWORDS_BASE.clone();
865 for w in PY_KEYWORDS.iter() {
866 combined.insert(w.clone());
867 }
868 for w in DOMAIN_STOPWORDS.iter() {
869 combined.insert(w.clone());
870 }
871 combined
872});
873
874static DOCS_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
875 set_from(&[
876 "the", "and", "for", "that", "with", "this", "from", "has", "have", "not", "are", "was",
877 "were", "will", "can", "you", "all", "any", "but", "they", "their", "there", "than",
878 "then", "when", "what", "which", "about", "after", "also", "been", "before", "between",
879 "both", "could", "each", "even", "into", "its", "just", "more", "most", "other", "over",
880 "same", "should", "some", "such", "through", "very", "well", "would", "use", "using",
881 "used", "may", "must", "only", "within", "without", "new", "old", "one", "two", "three",
882 "first", "last", "next", "many", "example", "note", "see", "get", "set", "add", "like",
883 "need", "heading", "section", "content", "long", "under", "value", "field", "name", "type",
884 "page", "text", "data", "item", "list", "file", "code", "function", "class", "method",
885 "return", "where", "how", "why", "output", "input", "result", "error", "default", "true",
886 "false",
887 ])
888});
889
890pub const PROFILE_CODE: &str = "code";
891pub const PROFILE_DOCS: &str = "docs";
892pub const PROFILE_LEGAL: &str = "legal";
893pub const PROFILE_DATA: &str = "data";
894pub const PROFILE_GENERIC: &str = "generic";
895
896static EMPTY_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(FxHashSet::default);
897
898pub fn get_stopwords(profile: &str) -> &FxHashSet<String> {
899 match profile {
900 PROFILE_CODE | PROFILE_GENERIC => &CODE_STOPWORDS,
901 PROFILE_DOCS | PROFILE_LEGAL => &DOCS_STOPWORDS,
902 PROFILE_DATA => &EMPTY_STOPWORDS,
903 _ => &CODE_STOPWORDS,
904 }
905}
906
907pub fn get_min_len(profile: &str) -> usize {
908 match profile {
909 PROFILE_CODE | PROFILE_DOCS | PROFILE_GENERIC => 3,
910 PROFILE_LEGAL => 4,
911 PROFILE_DATA => 2,
912 _ => 3,
913 }
914}
915
916pub fn profile_from_path(path: &str) -> &'static str {
917 let p = std::path::Path::new(path);
918 let suffix = p
919 .extension()
920 .map(|e| format!(".{}", e.to_string_lossy().to_lowercase()))
921 .unwrap_or_default();
922 let name_lower = p
923 .file_name()
924 .map(|n| n.to_string_lossy().to_lowercase())
925 .unwrap_or_default();
926
927 if CODE_EXTENSIONS.contains(suffix.as_str()) {
928 return PROFILE_CODE;
929 }
930
931 if DOC_EXTENSIONS.contains(suffix.as_str()) || suffix == ".markdown" || suffix == ".tex" {
932 return PROFILE_DOCS;
933 }
934
935 let data_exts = [
936 ".csv", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".xml", ".ini", ".env",
937 ];
938 if data_exts.contains(&suffix.as_str()) {
939 return PROFILE_DATA;
940 }
941
942 let stem = p
943 .file_stem()
944 .map(|s| s.to_string_lossy().to_lowercase())
945 .unwrap_or_default();
946 let legal_names = [
947 "license",
948 "licence",
949 "legal",
950 "terms",
951 "agreement",
952 "contract",
953 "policy",
954 "privacy",
955 "tos",
956 "eula",
957 ];
958 if legal_names.contains(&stem.as_str())
959 || name_lower.contains("license")
960 || name_lower.contains("legal")
961 || name_lower.contains("terms")
962 {
963 return PROFILE_LEGAL;
964 }
965
966 PROFILE_GENERIC
967}
968
969pub fn is_reasonable_ident(ident: &str, min_len: usize, profile: &str) -> bool {
970 if ident.is_empty() || ident.len() < min_len {
971 return false;
972 }
973 let low = ident.to_lowercase();
974 let stopwords = get_stopwords(profile);
975 if stopwords.contains(&low) {
976 return false;
977 }
978 if low.chars().all(|c| c.is_ascii_digit()) {
979 return false;
980 }
981 true
982}
983
984pub fn filter_idents(idents: &[String], min_len: usize, profile: &str) -> Vec<String> {
985 idents
986 .iter()
987 .filter(|s| is_reasonable_ident(s, min_len, profile))
988 .cloned()
989 .collect()
990}