use std::{
collections::{BTreeMap, BTreeSet, HashMap},
io::{self, Read},
path::{Path, PathBuf},
sync::{Arc, Mutex, PoisonError, RwLock},
};
use base64::{Engine, prelude::BASE64_STANDARD};
use cap_std::{ambient_authority, fs::Dir};
use crate::{
error::ErrorData as McpError,
model::{
DIRECTORY_MIME_TYPE, GetSkillResult, ListSkillsResult, ReadResourceDirectoryResult,
ReadResourceResult, Resource, ResourceContents, SKILL_MD, Skill, SkillFrontmatter,
SkillResource, SkillResources,
},
};
pub const SKILL_FILE_LIMIT: usize = 512;
pub const SKILL_BYTE_LIMIT: u64 = 16 * 1024 * 1024;
const URI_SCHEME: &str = "skill://";
const MARKDOWN_MIME_TYPE: &str = "text/markdown";
#[derive(Debug)]
pub struct SkillCatalog {
root: PathBuf,
snapshot: RwLock<Arc<Snapshot>>,
reloading: Mutex<()>,
}
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum SkillCatalogError {
#[error("cannot read {path}: {source}")]
Io {
path: PathBuf,
#[source]
source: io::Error,
},
}
impl SkillCatalog {
pub fn from_dir(root: impl Into<PathBuf>) -> Result<Self, SkillCatalogError> {
let root = root.into();
let snapshot = Snapshot::load(&root)?;
Ok(Self {
root,
snapshot: RwLock::new(Arc::new(snapshot)),
reloading: Mutex::new(()),
})
}
pub fn reload(&self) -> Result<bool, SkillCatalogError> {
let _reloading = self
.reloading
.lock()
.unwrap_or_else(PoisonError::into_inner);
let snapshot = Arc::new(Snapshot::load(&self.root)?);
let mut current = self
.snapshot
.write()
.unwrap_or_else(PoisonError::into_inner);
let changed = current.skills != snapshot.skills;
*current = snapshot;
Ok(changed)
}
pub fn skills(&self) -> Vec<Skill> {
self.current().skills.clone()
}
pub fn list_skills(&self) -> ListSkillsResult {
ListSkillsResult::with_all_items(self.skills())
}
pub fn get_skill(&self, uri: &str) -> Result<GetSkillResult, McpError> {
self.current()
.skills
.iter()
.find(|skill| skill.uri == uri)
.map(|skill| GetSkillResult::new(skill.clone()))
.ok_or_else(|| McpError::invalid_params(format!("unknown skill {uri}"), None))
}
pub fn resources(&self) -> Vec<Resource> {
self.current().listed.clone()
}
pub fn read_resource(&self, uri: &str) -> Option<ReadResourceResult> {
let snapshot = self.current();
let file = snapshot.files.get(uri)?;
let text = file
.mime_type
.is_some_and(is_textual)
.then(|| std::str::from_utf8(&file.bytes).ok())
.flatten();
let contents = match text {
Some(text) => ResourceContents::text(text, uri),
None => ResourceContents::blob(BASE64_STANDARD.encode(&file.bytes), uri),
};
let contents = match file.mime_type {
Some(mime_type) => contents.with_mime_type(mime_type),
None => contents,
};
Some(ReadResourceResult::new(vec![contents]))
}
pub fn read_resource_directory(
&self,
uri: &str,
) -> Result<ReadResourceDirectoryResult, McpError> {
self.current()
.directories
.get(uri)
.map(|children| ReadResourceDirectoryResult::with_all_items(children.clone()))
.ok_or_else(|| McpError::invalid_params(format!("not a directory: {uri}"), None))
}
fn current(&self) -> Arc<Snapshot> {
self.snapshot
.read()
.unwrap_or_else(PoisonError::into_inner)
.clone()
}
}
#[derive(Debug, Default)]
struct Snapshot {
skills: Vec<Skill>,
listed: Vec<Resource>,
files: HashMap<String, CatalogFile>,
directories: HashMap<String, Vec<Resource>>,
}
#[derive(Debug)]
struct CatalogFile {
bytes: Arc<[u8]>,
mime_type: Option<&'static str>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Kind {
File,
Directory,
Unsupported,
}
#[derive(Debug)]
struct Entry {
path: String,
kind: Kind,
size: u64,
}
impl Snapshot {
fn load(root: &Path) -> Result<Self, SkillCatalogError> {
let handle = Dir::open_ambient_dir(root, ambient_authority()).map_err(|source| {
SkillCatalogError::Io {
path: root.to_path_buf(),
source,
}
})?;
let tree = walk(root, &handle)?;
let skill_dirs: BTreeSet<&str> = tree
.iter()
.filter(|entry| is_manifest(entry))
.map(|entry| parent(&entry.path))
.collect();
let mut inside: BTreeMap<&str, Vec<&Entry>> = BTreeMap::new();
for entry in &tree {
let mut in_skill = false;
let mut dir = parent(&entry.path);
while !dir.is_empty() {
if skill_dirs.contains(dir) {
in_skill = true;
inside.entry(dir).or_default().push(entry);
}
dir = parent(dir);
}
if !in_skill && entry.kind == Kind::Unsupported && !is_manifest(entry) {
tracing::warn!(path = %entry.path, "skipping a symlink or special file");
}
}
let mut kept: Vec<(&str, &[&Entry])> = Vec::new();
for dir in skill_dirs {
if dir.is_empty() {
tracing::warn!("skipping the SKILL.md at the catalog root; a skill needs a folder");
continue;
}
let entries = inside.get(dir).map(Vec::as_slice).unwrap_or_default();
match check_walked(entries) {
Ok(()) => kept.push((dir, entries)),
Err(reason) => tracing::warn!(skill = %dir, "skipping skill: {reason}"),
}
}
let mut bytes_of: HashMap<&str, Arc<[u8]>> = HashMap::new();
let mut within_limit: Vec<(&str, &[&Entry])> = Vec::new();
for (dir, entries) in kept {
if read_within_limit(&handle, root, entries, &mut bytes_of)? {
within_limit.push((dir, entries));
} else {
tracing::warn!(
skill = %dir,
"skipping skill: grew past {SKILL_BYTE_LIMIT} bytes after the walk"
);
}
}
let mut snapshot = Snapshot::default();
let mut directories: BTreeSet<&str> = BTreeSet::new();
for (dir, entries) in within_limit {
let (skill, listing) = match build_skill(dir, entries, &bytes_of) {
Ok(built) => built,
Err(reason) => {
tracing::warn!(skill = %dir, "skipping skill: {reason}");
continue;
}
};
directories.insert(dir);
for entry in entries {
match entry.kind {
Kind::Directory => {
directories.insert(&entry.path);
}
Kind::File => {
snapshot
.files
.entry(uri_of(&entry.path))
.or_insert_with(|| CatalogFile {
bytes: Arc::clone(&bytes_of[entry.path.as_str()]),
mime_type: mime_type_of(&entry.path),
});
}
Kind::Unsupported => {}
}
}
snapshot.skills.push(skill);
snapshot.listed.push(listing);
}
snapshot.skills.sort_by(|a, b| a.uri.cmp(&b.uri));
snapshot.listed.sort_by(|a, b| a.uri.cmp(&b.uri));
let mut children: HashMap<&str, Vec<&Entry>> = HashMap::new();
for entry in &tree {
children.entry(parent(&entry.path)).or_default().push(entry);
}
for dir in directories {
let listing = children
.get(dir)
.into_iter()
.flatten()
.filter_map(|entry| directory_child(entry, &snapshot.files))
.collect();
snapshot.directories.insert(uri_of(dir), listing);
}
Ok(snapshot)
}
}
fn directory_child(entry: &Entry, files: &HashMap<String, CatalogFile>) -> Option<Resource> {
let uri = uri_of(&entry.path);
let name = file_name(&entry.path);
match entry.kind {
Kind::Directory => Some(Resource::new(uri, name).with_mime_type(DIRECTORY_MIME_TYPE)),
Kind::File => {
let file = files.get(&uri)?;
let mut resource = Resource::new(uri, name).with_size(file.bytes.len() as u64);
resource.mime_type = file.mime_type.map(Into::into);
Some(resource)
}
Kind::Unsupported => None,
}
}
fn check_walked(entries: &[&Entry]) -> Result<(), String> {
if let Some(entry) = entries.iter().find(|entry| entry.kind == Kind::Unsupported) {
return Err(format!(
"{} is a symlink, a special file, or not UTF-8; copy the file instead",
entry.path
));
}
let files = entries.iter().filter(|entry| entry.kind == Kind::File);
let (files, bytes) = (
files.clone().count(),
files.map(|entry| entry.size).sum::<u64>(),
);
if files > SKILL_FILE_LIMIT || bytes > SKILL_BYTE_LIMIT {
return Err(format!(
"{files} files and {bytes} bytes exceed the limit of {SKILL_FILE_LIMIT} files or \
{SKILL_BYTE_LIMIT} bytes, so hosts need not load it"
));
}
Ok(())
}
fn read_within_limit<'a>(
handle: &Dir,
root: &Path,
entries: &[&'a Entry],
bytes_of: &mut HashMap<&'a str, Arc<[u8]>>,
) -> Result<bool, SkillCatalogError> {
let mut budget = SKILL_BYTE_LIMIT + 1;
for entry in entries.iter().filter(|entry| entry.kind == Kind::File) {
let len = match bytes_of.get(entry.path.as_str()) {
Some(bytes) => bytes.len() as u64,
None => {
let mut bytes = Vec::new();
handle
.open(&entry.path)
.and_then(|file| file.take(budget).read_to_end(&mut bytes))
.map_err(|source| SkillCatalogError::Io {
path: root.join(&entry.path),
source,
})?;
let len = bytes.len() as u64;
if len < budget {
bytes_of.insert(&entry.path, bytes.into());
}
len
}
};
budget = budget.saturating_sub(len);
if budget == 0 {
return Ok(false);
}
}
Ok(true)
}
fn build_skill(
dir: &str,
entries: &[&Entry],
bytes_of: &HashMap<&str, Arc<[u8]>>,
) -> Result<(Skill, Resource), String> {
let manifest_path = format!("{dir}/{SKILL_MD}");
let manifest = bytes_of
.get(manifest_path.as_str())
.ok_or_else(|| format!("{manifest_path} is not a regular file"))?;
let frontmatter = parse_frontmatter(manifest)?;
let mut files: Vec<SkillResource> = entries
.iter()
.filter(|entry| entry.kind == Kind::File)
.map(|entry| SkillResource::from_bytes(uri_of(&entry.path), &bytes_of[entry.path.as_str()]))
.collect();
files.sort_by(|a, b| a.uri.cmp(&b.uri));
let uri = uri_of(&manifest_path);
let listing = Resource::new(&uri, &frontmatter.name)
.with_description(&frontmatter.description)
.with_mime_type(MARKDOWN_MIME_TYPE)
.with_size(manifest.len() as u64);
let skill = Skill::new(uri, frontmatter, SkillResources::Files(files));
skill.validate().map_err(|error| error.to_string())?;
Ok((skill, listing))
}
fn parse_frontmatter(bytes: &[u8]) -> Result<SkillFrontmatter, String> {
let text = std::str::from_utf8(bytes).map_err(|_| "SKILL.md is not UTF-8".to_string())?;
let mut lines = text.split_inclusive('\n');
if lines.next().map(str::trim_end) != Some("---") {
return Err("SKILL.md does not start with a --- frontmatter block".into());
}
let start = text.find('\n').map_or(text.len(), |index| index + 1);
let mut end = start;
for line in lines {
if line.trim_end() == "---" {
return yaml_serde::from_str(&text[start..end])
.map_err(|error| format!("invalid SKILL.md frontmatter: {error}"));
}
end += line.len();
}
Err("SKILL.md frontmatter has no closing ---".into())
}
fn walk(root: &Path, handle: &Dir) -> Result<Vec<Entry>, SkillCatalogError> {
fn visit(
root: &Path,
handle: &Dir,
dir: &str,
tree: &mut Vec<Entry>,
) -> Result<(), SkillCatalogError> {
let io_error = |source| SkillCatalogError::Io {
path: root.join(dir),
source,
};
let mut entries = handle
.read_dir(if dir.is_empty() { "." } else { dir })
.map_err(io_error)?
.collect::<Result<Vec<_>, _>>()
.map_err(io_error)?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries {
let name = entry.file_name();
let (name, utf8) = match name.into_string() {
Ok(name) => (name, true),
Err(name) => (name.to_string_lossy().into_owned(), false),
};
if name.starts_with('.') {
continue;
}
let file_type = entry.file_type().map_err(io_error)?;
let kind = match (utf8, file_type.is_file(), file_type.is_dir()) {
(true, true, _) => Kind::File,
(true, _, true) => Kind::Directory,
_ => Kind::Unsupported,
};
let path = if dir.is_empty() {
name
} else {
format!("{dir}/{name}")
};
if kind == Kind::Directory {
tree.push(Entry {
path: path.clone(),
kind,
size: 0,
});
visit(root, handle, &path, tree)?;
} else {
let size = match kind {
Kind::File => entry.metadata().map_err(io_error)?.len(),
_ => 0,
};
tree.push(Entry { path, kind, size });
}
}
Ok(())
}
let mut tree = Vec::new();
visit(root, handle, "", &mut tree)?;
Ok(tree)
}
fn is_manifest(entry: &Entry) -> bool {
entry.kind != Kind::Directory && file_name(&entry.path) == SKILL_MD
}
fn parent(path: &str) -> &str {
path.rsplit_once('/').map_or("", |(parent, _)| parent)
}
fn file_name(path: &str) -> &str {
path.rsplit_once('/').map_or(path, |(_, name)| name)
}
fn uri_of(path: &str) -> String {
let mut uri = String::from(URI_SCHEME);
for byte in path.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~' | b'/') {
uri.push(byte as char);
} else {
uri.push_str(&format!("%{byte:02X}"));
}
}
uri
}
fn is_textual(mime_type: &str) -> bool {
mime_type.starts_with("text/")
|| matches!(
mime_type,
"application/json"
| "application/xml"
| "application/yaml"
| "application/toml"
| "application/x-sh"
| "image/svg+xml"
)
}
fn mime_type_of(path: &str) -> Option<&'static str> {
let extension = file_name(path)
.rsplit_once('.')
.map(|(_, extension)| extension.to_ascii_lowercase());
let mime_type = match extension.as_deref() {
Some("md" | "markdown") => MARKDOWN_MIME_TYPE,
Some("txt") => "text/plain",
Some("html" | "htm") => "text/html",
Some("css") => "text/css",
Some("csv") => "text/csv",
Some("xml") => "application/xml",
Some("json") => "application/json",
Some("yaml" | "yml") => "application/yaml",
Some("toml") => "application/toml",
Some("js" | "mjs") => "text/javascript",
Some("ts") => "text/x-typescript",
Some("py") => "text/x-python",
Some("rs") => "text/x-rust",
Some("sh") => "application/x-sh",
Some("png") => "image/png",
Some("jpg" | "jpeg") => "image/jpeg",
Some("gif") => "image/gif",
Some("svg") => "image/svg+xml",
Some("webp") => "image/webp",
Some("pdf") => "application/pdf",
Some("zip") => "application/zip",
_ => return None,
};
Some(mime_type)
}
#[cfg(test)]
mod tests {
use std::fs::File;
use super::*;
#[test]
fn read_within_limit_should_stop_at_the_budget_and_not_cache_the_truncated_file() {
let dir = tempfile::tempdir().unwrap();
for name in ["a", "b"] {
File::create(dir.path().join(name))
.unwrap()
.set_len(SKILL_BYTE_LIMIT)
.unwrap();
}
let stale = |name: &str| Entry {
path: name.to_string(),
kind: Kind::File,
size: 1,
};
let (a, b) = (stale("a"), stale("b"));
let handle = Dir::open_ambient_dir(dir.path(), ambient_authority()).unwrap();
let mut bytes_of = HashMap::new();
let within = read_within_limit(&handle, dir.path(), &[&a, &b], &mut bytes_of).unwrap();
assert!(!within);
assert_eq!(bytes_of.keys().collect::<Vec<_>>(), vec![&"a"]);
}
}