use once_cell::sync::Lazy;
use rustc_hash::FxHashSet;
use crate::config::extensions::{CODE_EXTENSIONS, DOC_EXTENSIONS};
fn set_from(words: &[&str]) -> FxHashSet<String> {
words.iter().map(|w| w.to_string()).collect()
}
static PY_KEYWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
set_from(&[
"and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del",
"elif", "else", "except", "false", "finally", "for", "from", "global", "if", "import",
"in", "is", "lambda", "none", "nonlocal", "not", "or", "pass", "raise", "return", "true",
"try", "while", "with", "yield",
])
});
static CODE_STOPWORDS_BASE: Lazy<FxHashSet<String>> = Lazy::new(|| {
set_from(&[
"todo",
"fixme",
"note",
"hack",
"xxx",
"foo",
"bar",
"baz",
"qux",
"tmp",
"temp",
"self",
"cls",
"get",
"set",
"put",
"delete",
"remove",
"update",
"create",
"read",
"write",
"open",
"close",
"start",
"stop",
"run",
"execute",
"call",
"invoke",
"handle",
"process",
"validate",
"check",
"test",
"init",
"setup",
"teardown",
"load",
"save",
"parse",
"format",
"convert",
"transform",
"render",
"build",
"make",
"copy",
"clone",
"reset",
"clear",
"flush",
"send",
"receive",
"emit",
"insert",
"append",
"extend",
"split",
"join",
"trim",
"strip",
"contains",
"includes",
"push",
"pop",
"concat",
"slice",
"each",
"print",
"println",
"printf",
"sprintf",
"fprintf",
"abort",
"retry",
"apply",
"resolve",
"reject",
"defer",
"sleep",
"wait",
"poll",
"encode",
"decode",
"encrypt",
"decrypt",
"hash",
"sign",
"verify",
"register",
"unregister",
"mount",
"unmount",
"enable",
"disable",
"show",
"hide",
"lock",
"unlock",
"acquire",
"release",
"commit",
"rollback",
"begin",
"end",
"value",
"values",
"result",
"results",
"item",
"items",
"element",
"elements",
"key",
"keys",
"text",
"num",
"number",
"count",
"index",
"idx",
"size",
"length",
"len",
"array",
"dict",
"obj",
"object",
"args",
"kwargs",
"params",
"options",
"err",
"error",
"errors",
"msg",
"message",
"messages",
"log",
"logger",
"debug",
"info",
"warn",
"warning",
"func",
"function",
"method",
"callback",
"handler",
"listener",
"event",
"events",
"request",
"response",
"req",
"res",
"body",
"header",
"headers",
"query",
"param",
"id",
"uid",
"uuid",
"var",
"let",
"const",
"new",
"this",
"null",
"nil",
"none",
"void",
"true",
"false",
"static",
"final",
"abstract",
"virtual",
"override",
"public",
"private",
"protected",
"internal",
"extern",
"inline",
"volatile",
"mutable",
"readonly",
"super",
"extends",
"implements",
"import",
"export",
"require",
"include",
"using",
"typedef",
"define",
"ifdef",
"ifndef",
"endif",
"elif",
"else",
"case",
"switch",
"break",
"continue",
"return",
"goto",
"sizeof",
"typeof",
"instanceof",
"throw",
"throws",
"try",
"catch",
"finally",
"raise",
"except",
"yield",
"lambda",
"proc",
"sub",
"def",
"async",
"await",
"int",
"uint",
"long",
"float",
"double",
"char",
"byte",
"short",
"signed",
"unsigned",
"bool",
"boolean",
"string",
"str",
"size_t",
"usize",
"isize",
"int8",
"int16",
"int32",
"int64",
"uint8",
"uint16",
"uint32",
"uint64",
"float32",
"float64",
"__init__",
"__main__",
"__name__",
"__str__",
"__repr__",
"__eq__",
"__hash__",
"__len__",
"__iter__",
"__next__",
"__enter__",
"__exit__",
"echo",
"done",
"local",
"eval",
"exec",
"exit",
"trap",
"bash",
"then",
"esac",
"fi",
"do",
"for",
"while",
"until",
"shift",
"unset",
"declare",
"alias",
"html",
"head",
"div",
"span",
"form",
"button",
"img",
"href",
"style",
"display",
"margin",
"padding",
"border",
"color",
"width",
"height",
"flex",
"grid",
"onclick",
"onchange",
"onsubmit",
"classname",
])
});
static DOMAIN_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
set_from(&[
"add",
"sort",
"filter",
"find",
"select",
"merge",
"match",
"replace",
"dispatch",
"notify",
"subscribe",
"publish",
"connect",
"disconnect",
"bind",
"listen",
"accept",
"every",
"some",
"reduce",
"splice",
"unshift",
"throw",
"catch",
"create",
"destroy",
"dispose",
"finalize",
"data",
"name",
"names",
"list",
"map",
"config",
"settings",
"context",
"ctx",
"state",
"status",
"type",
"kind",
"mode",
"flag",
"flags",
"path",
"file",
"dir",
"url",
"uri",
"host",
"port",
"input",
"output",
"source",
"target",
"dest",
"src",
"dst",
"user",
"users",
"model",
"models",
"view",
"views",
"service",
"services",
"client",
"server",
"api",
"app",
"main",
"util",
"utils",
"helper",
"helpers",
"common",
"base",
"core",
"default",
"defaults",
"version",
"label",
"labels",
"tag",
"tags",
"level",
"scope",
"token",
"tokens",
"task",
"tasks",
"job",
"jobs",
"step",
"steps",
"stage",
"cache",
"timeout",
"interval",
"duration",
"timestamp",
"channel",
"buffer",
"queue",
"stack",
"heap",
"node",
"nodes",
"edge",
"edges",
"child",
"children",
"parent",
"root",
"leaf",
"link",
"ref",
"reference",
"instance",
"schema",
"table",
"column",
"row",
"field",
"record",
"entry",
"spec",
"env",
"namespace",
"prefix",
"suffix",
"pattern",
"template",
"factory",
"builder",
"adapter",
"proxy",
"wrapper",
"manager",
"controller",
"provider",
"consumer",
"producer",
"worker",
"pool",
"connection",
"session",
"stream",
"pipe",
"socket",
"signal",
"trigger",
"action",
"command",
"rule",
"rules",
"policy",
"strategy",
"middleware",
"plugin",
"module",
"package",
"component",
"container",
"registry",
"repository",
"interface",
"struct",
"enum",
"union",
"trait",
"impl",
"class",
"implement",
"inherit",
"mixin",
"tostring",
"hashcode",
"equals",
"usestate",
"useeffect",
"usecontext",
"usereducer",
"usecallback",
"usememo",
"useref",
"uselayouteffect",
"useimperativehandle",
"usedebugvalue",
"useid",
"usetransition",
"usedeferredvalue",
"createcontext",
"forwardref",
"createref",
"suspense",
"strictmode",
"profiler",
"usenavigate",
"useparams",
"uselocation",
"usesearchparams",
"useloaderdata",
"useactiondata",
"usefetcher",
"useoutletcontext",
"usedispatch",
"useselector",
"usestore",
"usequery",
"usemutation",
"usesubscription",
"definecomponent",
"defineprops",
"defineemits",
"defineslots",
"definemodel",
"defineexpose",
"toref",
"torefs",
"reactive",
"computed",
"onmounted",
"onunmounted",
"onbeforemount",
"onupdated",
"watcheffect",
"nexttick",
"ngoninit",
"ngondestroy",
"ngonchanges",
"ngafterviewinit",
"console",
"document",
"window",
"navigator",
"location",
"history",
"fetch",
"promise",
"undefined",
"prototype",
"constructor",
"exports",
"props",
"effect",
"memo",
"styles",
"innerhtml",
"textcontent",
"appendchild",
"createelement",
"queryselector",
"getelementbyid",
"addeventlistener",
"resource",
"variable",
"locals",
"terraform",
"jsonencode",
"cidr",
"subnet",
"ingress",
"egress",
"protocol",
"cidr_blocks",
"security_group",
"arn",
"vpc",
"aws",
"gcp",
"azure",
"region",
"zone",
"cluster",
"replicas",
"selector",
"metadata",
"annotations",
"volumes",
"ports",
"image",
"containers",
"resources",
"limits",
"requests",
"apiversion",
"configmap",
"secret",
"deployment",
"statefulset",
"daemonset",
"cronjob",
"serviceaccount",
"role",
"rolebinding",
"clusterrole",
"install",
"deploy",
"pipeline",
"script",
"pull",
"uses",
"with",
"needs",
"runs",
"checkout",
"artifact",
"artifacts",
"workflow",
"schedule",
"cron",
"branch",
"branches",
"release",
"from",
"workdir",
"expose",
"entrypoint",
"volume",
"arg",
"healthcheck",
"stopsignal",
"describe",
"it",
"expect",
"assert",
"should",
"mock",
"stub",
"spy",
"before",
"after",
"beforeall",
"afterall",
"beforeeach",
"aftereach",
"suite",
"fixture",
"given",
"when",
"insert",
"update",
"delete",
"from",
"where",
"join",
"inner",
"outer",
"left",
"right",
"group",
"order",
"limit",
"offset",
"having",
"distinct",
"count",
"sum",
"avg",
"min",
"max",
"constraint",
"primary",
"foreign",
"unique",
"not",
"and",
"or",
"between",
"exists",
"values",
"into",
"alter",
"drop",
"truncate",
"grant",
"revoke",
"body",
"title",
"meta",
"script",
"label",
"select",
"option",
"thead",
"tbody",
"class",
"type",
"name",
"value",
"placeholder",
"required",
"disabled",
"checked",
"selected",
"hidden",
"http",
"https",
"tcp",
"udp",
"dns",
"ssl",
"tls",
"cert",
"certificate",
"content",
"authorization",
"bearer",
"basic",
"digest",
"origin",
"referer",
"cookie",
"cookies",
"redirect",
"gateway",
"load",
"balance",
"upstream",
"downstream",
"endpoint",
"route",
"routes",
"router",
"method",
"code",
"java",
"python",
"golang",
"ruby",
"rust",
"swift",
"kotlin",
"scala",
"perl",
"elixir",
"haskell",
"clojure",
"erlang",
"typescript",
"javascript",
"csharp",
"cplusplus",
"alpine",
"ubuntu",
"debian",
"linux",
"darwin",
"windows",
"time",
"math",
"system",
"vector",
"hashmap",
"hashset",
"treemap",
"arraylist",
"linkedlist",
"optional",
"future",
"pair",
"tuple",
"collection",
"collections",
"iterator",
"iterable",
"comparable",
"serializable",
"cloneable",
"runnable",
"callable",
"supplier",
"predicate",
"comparator",
"json",
"xml",
"yaml",
"toml",
"csv",
"email",
"phone",
"address",
"date",
"datetime",
"password",
"username",
"auth",
"login",
"logout",
"admin",
"account",
"profile",
"permission",
"team",
"organization",
"project",
"description",
"category",
"comment",
"comments",
"like",
"share",
"search",
"page",
"pages",
"upload",
"download",
"notification",
"notifications",
"alert",
"alerts",
"validation",
"production",
"staging",
"development",
"testing",
"localhost",
"unknown",
"refs",
"heads",
"remote",
"master",
"rebase",
"cherry",
"stash",
])
});
pub static CODE_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
let mut combined = CODE_STOPWORDS_BASE.clone();
for w in PY_KEYWORDS.iter() {
combined.insert(w.clone());
}
for w in DOMAIN_STOPWORDS.iter() {
combined.insert(w.clone());
}
combined
});
static DOCS_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(|| {
set_from(&[
"the", "and", "for", "that", "with", "this", "from", "has", "have", "not", "are", "was",
"were", "will", "can", "you", "all", "any", "but", "they", "their", "there", "than",
"then", "when", "what", "which", "about", "after", "also", "been", "before", "between",
"both", "could", "each", "even", "into", "its", "just", "more", "most", "other", "over",
"same", "should", "some", "such", "through", "very", "well", "would", "use", "using",
"used", "may", "must", "only", "within", "without", "new", "old", "one", "two", "three",
"first", "last", "next", "many", "example", "note", "see", "get", "set", "add", "like",
"need", "heading", "section", "content", "long", "under", "value", "field", "name", "type",
"page", "text", "data", "item", "list", "file", "code", "function", "class", "method",
"return", "where", "how", "why", "output", "input", "result", "error", "default", "true",
"false",
])
});
pub const PROFILE_CODE: &str = "code";
pub const PROFILE_DOCS: &str = "docs";
pub const PROFILE_LEGAL: &str = "legal";
pub const PROFILE_DATA: &str = "data";
pub const PROFILE_GENERIC: &str = "generic";
static EMPTY_STOPWORDS: Lazy<FxHashSet<String>> = Lazy::new(FxHashSet::default);
pub fn get_stopwords(profile: &str) -> &FxHashSet<String> {
match profile {
PROFILE_CODE | PROFILE_GENERIC => &CODE_STOPWORDS,
PROFILE_DOCS | PROFILE_LEGAL => &DOCS_STOPWORDS,
PROFILE_DATA => &EMPTY_STOPWORDS,
_ => &CODE_STOPWORDS,
}
}
pub fn get_min_len(profile: &str) -> usize {
match profile {
PROFILE_CODE | PROFILE_DOCS | PROFILE_GENERIC => 3,
PROFILE_LEGAL => 4,
PROFILE_DATA => 2,
_ => 3,
}
}
pub fn profile_from_path(path: &str) -> &'static str {
let p = std::path::Path::new(path);
let suffix = p
.extension()
.map(|e| format!(".{}", e.to_string_lossy().to_lowercase()))
.unwrap_or_default();
let name_lower = p
.file_name()
.map(|n| n.to_string_lossy().to_lowercase())
.unwrap_or_default();
if CODE_EXTENSIONS.contains(suffix.as_str()) {
return PROFILE_CODE;
}
if DOC_EXTENSIONS.contains(suffix.as_str()) || suffix == ".markdown" || suffix == ".tex" {
return PROFILE_DOCS;
}
let data_exts = [
".csv", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".xml", ".ini", ".env",
];
if data_exts.contains(&suffix.as_str()) {
return PROFILE_DATA;
}
let stem = p
.file_stem()
.map(|s| s.to_string_lossy().to_lowercase())
.unwrap_or_default();
let legal_names = [
"license",
"licence",
"legal",
"terms",
"agreement",
"contract",
"policy",
"privacy",
"tos",
"eula",
];
if legal_names.contains(&stem.as_str())
|| name_lower.contains("license")
|| name_lower.contains("legal")
|| name_lower.contains("terms")
{
return PROFILE_LEGAL;
}
PROFILE_GENERIC
}
pub fn is_reasonable_ident(ident: &str, min_len: usize, profile: &str) -> bool {
if ident.is_empty() || ident.len() < min_len {
return false;
}
let low = ident.to_lowercase();
let stopwords = get_stopwords(profile);
if stopwords.contains(&low) {
return false;
}
if low.chars().all(|c| c.is_ascii_digit()) {
return false;
}
true
}
pub fn filter_idents(idents: &[String], min_len: usize, profile: &str) -> Vec<String> {
idents
.iter()
.filter(|s| is_reasonable_ident(s, min_len, profile))
.cloned()
.collect()
}