use std::collections::BTreeMap;
use std::path::Path;
use murmur_artifact::PACKED_MANIFEST_ENTRY;
use serde_json::Value;
use crate::protected_paths::{matches_key, TOOL_DESTINATION_KEYS, TOOL_PATH_KEYS};
pub(crate) const FORMAT_DESTINATION: &str = "murmur-destination";
pub(crate) const FORMAT_OPAQUE: &str = "murmur-opaque";
pub(crate) const KEYWORD_DESTINATIONS: &str = "murmur-destinations";
const MAX_SCHEMA_DEPTH: usize = 32;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(crate) enum LocationStep {
Key(String),
Element,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(crate) struct InputLocation {
steps: Vec<LocationStep>,
}
impl InputLocation {
fn child(&self, step: LocationStep) -> Self {
let mut steps = self.steps.clone();
steps.push(step);
Self { steps }
}
pub(crate) fn render(&self) -> String {
if self.steps.is_empty() {
return "<input>".to_string();
}
let mut out = String::new();
for step in &self.steps {
match step {
LocationStep::Key(key) => {
if !out.is_empty() {
out.push('.');
}
out.push_str(key);
}
LocationStep::Element => out.push_str("[]"),
}
}
out
}
fn parse(text: &str) -> Option<Self> {
if text == "<input>" {
return Some(Self::default());
}
if text.is_empty() {
return None;
}
let mut steps = Vec::new();
for segment in text.split('.') {
let split = segment.find('[').unwrap_or(segment.len());
let (key, mut rest) = segment.split_at(split);
if key.is_empty() || key.contains(']') {
return None;
}
steps.push(LocationStep::Key(key.to_string()));
while let Some(tail) = rest.strip_prefix("[]") {
steps.push(LocationStep::Element);
rest = tail;
}
if !rest.is_empty() {
return None;
}
}
Some(Self { steps })
}
pub(crate) fn resolve<'a>(&self, input: &'a Value) -> Vec<&'a str> {
let mut out = Vec::new();
collect_at(input, &self.steps, &mut out);
out
}
}
fn collect_at<'a>(value: &'a Value, steps: &[LocationStep], out: &mut Vec<&'a str>) {
let Some((step, rest)) = steps.split_first() else {
if let Some(text) = value.as_str() {
out.push(text);
}
return;
};
match (step, value) {
(LocationStep::Key(key), Value::Object(map)) => {
if let Some(child) = map.get(key) {
collect_at(child, rest, out);
}
}
(LocationStep::Element, Value::Array(items)) => {
for item in items {
collect_at(item, rest, out);
}
}
_ => {}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DerivedDestination {
base: InputLocation,
suffix: Option<String>,
declared: String,
}
impl DerivedDestination {
fn parse(entry: &str, visited: &[InputLocation]) -> Option<Self> {
if entry.contains('\\') {
return None;
}
let rest = entry.strip_prefix('{')?;
let (location, tail) = rest.split_once('}')?;
let base = InputLocation::parse(location)?;
if !visited.contains(&base) {
return None;
}
let suffix = match tail {
"" => None,
_ => {
let relative = tail.strip_prefix('/')?;
if relative
.split('/')
.any(|c| c.is_empty() || c == "." || c == "..")
{
return None;
}
Some(relative.to_string())
}
};
Some(Self {
base,
suffix,
declared: entry.to_string(),
})
}
pub(crate) fn declared(&self) -> &str {
&self.declared
}
pub(crate) fn candidates(&self, input: &Value) -> Vec<String> {
self.base
.resolve(input)
.into_iter()
.filter_map(|base| match (&self.suffix, base) {
(Some(suffix), "") => Some(suffix.clone()),
(Some(suffix), base) => Some(format!("{base}/{suffix}")),
(None, "") => None,
(None, base) => Some(base.to_string()),
})
.collect()
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct ToolAnnotations {
destinations: Vec<InputLocation>,
opaque: Vec<InputLocation>,
derived: Vec<DerivedDestination>,
}
impl ToolAnnotations {
pub(crate) fn from_schema_json(schema: &str) -> Self {
LoweredSchema::from_schema_json(schema).annotations
}
pub(crate) fn is_empty(&self) -> bool {
self.destinations.is_empty() && self.opaque.is_empty() && self.derived.is_empty()
}
pub(crate) fn destinations(&self) -> &[InputLocation] {
&self.destinations
}
pub(crate) fn derived(&self) -> &[DerivedDestination] {
&self.derived
}
pub(crate) fn is_opaque(&self, steps: &[LocationStep]) -> bool {
self.opaque.iter().any(|location| location.steps == steps)
}
}
struct PathShapedProperty {
name: String,
at: InputLocation,
container: bool,
}
struct LoweredSchema {
annotations: ToolAnnotations,
declares_destinations: bool,
visited: Vec<InputLocation>,
path_shaped: Vec<PathShapedProperty>,
}
impl LoweredSchema {
fn from_schema_json(schema: &str) -> Self {
let mut lowered = Self {
annotations: ToolAnnotations::default(),
declares_destinations: false,
visited: Vec::new(),
path_shaped: Vec::new(),
};
if let Ok(value) = serde_json::from_str::<Value>(schema) {
lowered.walk(&value, &InputLocation::default(), 0, false);
lowered.lower_root_destinations(&value);
}
lowered
}
fn walk(&mut self, node: &Value, at: &InputLocation, depth: usize, in_opaque: bool) {
if depth > MAX_SCHEMA_DEPTH {
return;
}
let Some(map) = node.as_object() else { return };
push_unique(&mut self.visited, at);
let format = murmur_format(node);
match format {
Some(FORMAT_DESTINATION) => push_unique(&mut self.annotations.destinations, at),
Some(FORMAT_OPAQUE) => push_unique(&mut self.annotations.opaque, at),
_ => {}
}
let in_opaque = in_opaque || format == Some(FORMAT_OPAQUE);
if let Some(properties) = map.get("properties").and_then(Value::as_object) {
for (key, child) in properties {
let location = at.child(LocationStep::Key(key.clone()));
if !in_opaque
&& (matches_key(TOOL_PATH_KEYS, key) || matches_key(TOOL_DESTINATION_KEYS, key))
{
self.path_shaped.push(PathShapedProperty {
name: key.clone(),
at: location.clone(),
container: declares_container(child),
});
}
self.walk(child, &location, depth + 1, in_opaque);
}
}
if let Some(items) = map.get("items") {
let element = at.child(LocationStep::Element);
match items {
Value::Array(entries) => {
for entry in entries {
self.walk(entry, &element, depth + 1, in_opaque);
}
}
other => self.walk(other, &element, depth + 1, in_opaque),
}
}
if let Some(Value::Array(entries)) = map.get("prefixItems") {
let element = at.child(LocationStep::Element);
for entry in entries {
self.walk(entry, &element, depth + 1, in_opaque);
}
}
for keyword in ["allOf", "anyOf", "oneOf"] {
if let Some(Value::Array(branches)) = map.get(keyword) {
for branch in branches {
self.walk(branch, at, depth + 1, in_opaque);
}
}
}
}
fn lower_root_destinations(&mut self, root: &Value) {
let Some(entries) = root.get(KEYWORD_DESTINATIONS) else {
return;
};
let Some(entries) = entries.as_array() else {
return;
};
let mut lowered = Vec::new();
for entry in entries {
let Some(derived) = entry
.as_str()
.and_then(|text| DerivedDestination::parse(text, &self.visited))
else {
return;
};
lowered.push(derived);
}
self.declares_destinations = true;
self.annotations.derived = lowered;
}
}
fn murmur_format(node: &Value) -> Option<&str> {
match node.get("format").and_then(Value::as_str) {
Some(value @ (FORMAT_DESTINATION | FORMAT_OPAQUE)) => Some(value),
_ => None,
}
}
fn declares_container(node: &Value) -> bool {
let container = |name: &Value| matches!(name.as_str(), Some("object" | "array"));
match node.get("type") {
Some(Value::Array(names)) => names.iter().any(container),
Some(name) => container(name),
None => false,
}
}
fn push_unique(locations: &mut Vec<InputLocation>, location: &InputLocation) {
if !locations.contains(location) {
locations.push(location.clone());
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct ToolAnnotationMap {
tools: BTreeMap<String, ToolAnnotations>,
}
impl ToolAnnotationMap {
pub(crate) fn from_workdir(workdir: &Path) -> Self {
let mut tools = BTreeMap::new();
let Ok(entries) = std::fs::read_dir(workdir.join("tools")) else {
return Self { tools };
};
for entry in entries.flatten() {
let path = entry.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let Some(schema) = read_staged_schema(&path.join(PACKED_MANIFEST_ENTRY)) else {
continue;
};
let annotations = ToolAnnotations::from_schema_json(&schema);
if !annotations.is_empty() {
tools.insert(name.to_string(), annotations);
}
}
Self { tools }
}
pub(crate) fn for_tool(&self, tool_name: &str) -> &ToolAnnotations {
static NONE: ToolAnnotations = ToolAnnotations {
destinations: Vec::new(),
opaque: Vec::new(),
derived: Vec::new(),
};
self.tools.get(tool_name).unwrap_or(&NONE)
}
#[cfg(test)]
pub(crate) fn from_schemas(entries: &[(&str, &str)]) -> Self {
Self {
tools: entries
.iter()
.map(|(name, schema)| (name.to_string(), ToolAnnotations::from_schema_json(schema)))
.collect(),
}
}
#[cfg(test)]
pub(crate) fn is_empty(&self) -> bool {
self.tools.is_empty()
}
}
pub(crate) fn declared_input_schema(manifest: &serde_yaml::Value) -> Option<Value> {
let schema = manifest
.get("input_schema")
.or_else(|| manifest.get("input"))?;
match schema {
serde_yaml::Value::String(text) => serde_json::from_str(text).ok(),
other => serde_json::to_value(other).ok(),
}
}
pub(crate) fn schema_from_manifest_yaml(manifest_yaml: &str) -> Option<String> {
let manifest: serde_yaml::Value = serde_yaml::from_str(manifest_yaml).ok()?;
Some(declared_input_schema(&manifest)?.to_string())
}
fn read_staged_schema(manifest_path: &Path) -> Option<String> {
schema_from_manifest_yaml(&std::fs::read_to_string(manifest_path).ok()?)
}
pub(crate) fn unannotated_path_properties(schema: &str) -> Vec<String> {
let lowered = LoweredSchema::from_schema_json(schema);
if lowered.declares_destinations {
return Vec::new();
}
let mut names: Vec<String> = Vec::new();
for property in lowered.path_shaped {
let answered = lowered.annotations.destinations.contains(&property.at)
|| (property.container && lowered.annotations.opaque.contains(&property.at));
if !answered && !names.contains(&property.name) {
names.push(property.name);
}
}
names
}
#[cfg(test)]
mod tests {
use super::*;
fn location(steps: &[LocationStep]) -> InputLocation {
InputLocation {
steps: steps.to_vec(),
}
}
fn key(name: &str) -> LocationStep {
LocationStep::Key(name.to_string())
}
#[test]
fn a_location_renders_with_dots_and_array_steps() {
assert_eq!(location(&[]).render(), "<input>");
assert_eq!(location(&[key("sink")]).render(), "sink");
assert_eq!(
location(&[key("edits"), LocationStep::Element, key("path")]).render(),
"edits[].path"
);
assert_eq!(
location(&[key("a"), LocationStep::Element, LocationStep::Element]).render(),
"a[][]"
);
}
#[test]
fn a_location_resolves_every_string_it_names() {
let input = serde_json::json!({
"edits": [{"path": "a.py"}, {"path": "b.py"}, {"path": {"nested": "c.py"}}, {}],
"sink": 7
});
let at = location(&[key("edits"), LocationStep::Element, key("path")]);
assert_eq!(at.resolve(&input), vec!["a.py", "b.py"]);
assert!(location(&[key("sink")]).resolve(&input).is_empty());
assert!(location(&[key("absent")]).resolve(&input).is_empty());
}
#[test]
fn the_walk_lowers_each_annotation_to_its_location() {
let annotations = ToolAnnotations::from_schema_json(
r#"{
"type": "object",
"properties": {
"edits": {
"type": "array",
"items": {
"type": "object",
"properties": {"path": {"type": "string", "format": "murmur-destination"}}
}
},
"note": {"type": "object", "format": "murmur-opaque"},
"pair": {"type": "array", "prefixItems": [
{"type": "string", "format": "murmur-destination"}
]},
"either": {"anyOf": [{"type": "object", "format": "murmur-opaque"}]}
}
}"#,
);
assert_eq!(
annotations.destinations(),
&[
location(&[key("edits"), LocationStep::Element, key("path")]),
location(&[key("pair"), LocationStep::Element]),
]
);
assert!(annotations.is_opaque(&[key("note")]));
assert!(
annotations.is_opaque(&[key("either")]),
"a composition branch describes the same location it sits under"
);
assert!(!annotations.is_opaque(&[key("edits")]));
}
#[test]
fn an_unknown_format_and_a_ref_lower_to_nothing() {
assert!(ToolAnnotations::from_schema_json(
r#"{"properties":{"path":{"type":"string","format":"uri-reference"}}}"#
)
.is_empty());
assert!(ToolAnnotations::from_schema_json(
r##"{"properties":{"path":{"$ref":"#/$defs/dest"}},"$defs":{"dest":{"format":"murmur-destination"}}}"##
)
.is_empty());
}
#[test]
fn an_unparsable_schema_lowers_to_nothing() {
for schema in ["", "not json at all", "[1,2,3]", "\"a string\""] {
assert!(
ToolAnnotations::from_schema_json(schema).is_empty(),
"{schema}"
);
}
}
#[test]
fn the_walk_stops_at_the_depth_bound() {
let mut schema = r#"{"type":"string","format":"murmur-destination"}"#.to_string();
for _ in 0..(MAX_SCHEMA_DEPTH + 10) {
schema = format!(r#"{{"type":"object","properties":{{"n":{schema}}}}}"#);
}
assert!(ToolAnnotations::from_schema_json(&schema).is_empty());
}
#[test]
fn a_location_parses_back_from_its_rendered_spelling() {
for steps in [
vec![],
vec![key("sink")],
vec![key("edits"), LocationStep::Element, key("path")],
vec![key("a"), LocationStep::Element, LocationStep::Element],
] {
let at = location(&steps);
assert_eq!(
InputLocation::parse(&at.render()),
Some(at.clone()),
"{at:?}"
);
}
for malformed in ["", ".", "a.", "a..b", "a[", "a]", "a[0]", "[]", "a[]b"] {
assert_eq!(InputLocation::parse(malformed), None, "{malformed}");
}
}
#[test]
fn a_derived_destination_entry_parses_only_in_its_grammar() {
let visited = vec![
location(&[]),
location(&[key("repo_path")]),
location(&[key("paths"), LocationStep::Element]),
];
let parsed = |entry: &str| DerivedDestination::parse(entry, &visited);
let derived = parsed("{repo_path}/.murmur").expect("a suffixed entry parses");
assert_eq!(derived.base, location(&[key("repo_path")]));
assert_eq!(derived.suffix.as_deref(), Some(".murmur"));
assert_eq!(derived.declared(), "{repo_path}/.murmur");
let bare = parsed("{repo_path}").expect("an entry with no suffix parses");
assert_eq!(bare.suffix, None);
assert!(
parsed("{paths[]}/out").is_some(),
"an array step is a location"
);
assert!(
parsed("{repo_path}/a/b/c").is_some(),
"a multi-component suffix is one relative path"
);
for malformed in [
"repo_path/.murmur",
"{repo_path",
"{repo_path}//etc",
"{repo_path}/",
"{repo_path}/../out",
"{repo_path}/./out",
"{repo_path}/a/../b",
"{repo_path}\\out",
"{repo_path}.murmur",
"{repo-path}/.murmur",
"{absent}",
"{}",
] {
assert!(parsed(malformed).is_none(), "{malformed}");
}
}
#[test]
fn a_declared_list_lowers_to_its_derived_destinations() {
let annotations = ToolAnnotations::from_schema_json(
r#"{"type":"object",
"properties":{"repo_path":{"type":"string"},"paths":{"type":"array","items":{"type":"string"}}},
"murmur-destinations":["{repo_path}/.murmur","{paths[]}"]}"#,
);
let input = serde_json::json!({"repo_path": "repo", "paths": ["a", "b"]});
let candidates: Vec<String> = annotations
.derived()
.iter()
.flat_map(|derived| derived.candidates(&input))
.collect();
assert_eq!(candidates, vec!["repo/.murmur", "a", "b"]);
assert_eq!(
annotations.derived()[0].declared(),
"{repo_path}/.murmur",
"a refusal names the entry as the tool wrote it"
);
assert_eq!(
annotations.derived()[0].candidates(&serde_json::json!({"repo_path": ""})),
vec![".murmur"],
"a location resolving to the empty string names the workdir root"
);
}
#[test]
fn one_malformed_entry_invalidates_the_whole_list() {
let schema = |list: &str| {
format!(
r#"{{"type":"object","properties":{{"repo_path":{{"type":"string"}},"file":{{"type":"string"}}}},"murmur-destinations":{list}}}"#
)
};
for list in [
r#""{repo_path}""#,
r#"[7]"#,
r#"["{repo_path}/.murmur","{repo_path}/../out"]"#,
r#"["repo_path/.murmur"]"#,
r#"["{repo-path}/.murmur"]"#,
] {
let text = schema(list);
assert!(
ToolAnnotations::from_schema_json(&text)
.derived()
.is_empty(),
"{list}"
);
assert_eq!(
unannotated_path_properties(&text),
vec!["file".to_string()],
"an unreadable declaration leaves the warning firing: {list}"
);
}
}
#[test]
fn a_nested_destinations_keyword_is_not_read() {
let annotations = ToolAnnotations::from_schema_json(
r#"{"type":"object","properties":{"inner":{"type":"object",
"properties":{"repo_path":{"type":"string"}},
"murmur-destinations":["{inner.repo_path}"]}}}"#,
);
assert!(annotations.derived().is_empty());
}
#[test]
fn the_warning_names_every_unannotated_path_shaped_property() {
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"file_path":{"type":"string"},"content":{"type":"string"}}}"#
),
vec!["file_path".to_string()]
);
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"dest":{"type":"string","format":"murmur-destination"},"repo":{"type":"string"},"path":{"type":"string"}}}"#
),
vec!["path".to_string()],
"an annotated sibling silences itself and nothing else"
);
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"edits":{"type":"array","items":{"type":"object","properties":{"filename":{"type":"string"}}}},"file":{"type":"string"}}}"#
),
vec!["filename".to_string(), "file".to_string()],
"a nested property is what the heuristic reads, so it is what the warning reads, in \
the order the walk reaches it"
);
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"query":{"type":"string"}}}"#
),
Vec::<String>::new()
);
assert!(unannotated_path_properties("").is_empty());
}
#[test]
fn a_property_inside_an_opaque_subtree_is_not_named() {
assert!(unannotated_path_properties(
r#"{"type":"object","properties":{"note":{"type":"object","format":"murmur-opaque","properties":{"file":{"type":"string"}}}}}"#
)
.is_empty());
}
#[test]
fn an_opaque_declaration_on_a_non_container_property_keeps_warning() {
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"path":{"type":"string","format":"murmur-opaque"}}}"#
),
vec!["path".to_string()],
"a string property is still judged by key name, whatever it declares"
);
assert_eq!(
unannotated_path_properties(
r#"{"type":"object","properties":{"file":{"format":"murmur-opaque"}}}"#
),
vec!["file".to_string()],
"a node declaring no type declares no container"
);
assert!(
unannotated_path_properties(
r#"{"type":"object","properties":{"file":{"type":"object","format":"murmur-opaque","properties":{"body":{"type":"string"}}}}}"#
)
.is_empty(),
"on a container the declaration is what stops the heuristic descending"
);
assert!(
unannotated_path_properties(
r#"{"type":"object","properties":{"file":{"type":["object","null"],"format":"murmur-opaque"}}}"#
)
.is_empty(),
"the type-list spelling declares the same container"
);
}
#[test]
fn a_valid_destination_list_silences_the_warning() {
assert!(unannotated_path_properties(
r#"{"type":"object","properties":{"repo_path":{"type":"string"},"file":{"type":"string"},"path":{"type":"string"}},"murmur-destinations":["{repo_path}/.murmur"]}"#
)
.is_empty());
assert!(
unannotated_path_properties(
r#"{"type":"object","properties":{"file":{"type":"string"}},"murmur-destinations":[]}"#
)
.is_empty(),
"the empty list declares that the tool writes nothing named by or derived from its input"
);
}
#[test]
fn a_staged_manifest_yields_its_schema_in_either_spelling() {
let from_string = schema_from_manifest_yaml(
"name: noter\ninput_schema: '{\"properties\":{\"note\":{\"format\":\"murmur-opaque\"}}}'\n",
)
.expect("a JSON string schema is read");
assert!(ToolAnnotations::from_schema_json(&from_string).is_opaque(&[key("note")]));
let from_mapping = schema_from_manifest_yaml(
"name: noter\ninput_schema:\n properties:\n note:\n format: murmur-opaque\n",
)
.expect("a YAML mapping schema is read");
assert!(ToolAnnotations::from_schema_json(&from_mapping).is_opaque(&[key("note")]));
assert!(schema_from_manifest_yaml("name: noter\nversion: 0.1.0\n").is_none());
}
#[test]
fn the_workdir_read_keys_annotations_by_tool_name() {
let workdir = tempfile::tempdir().unwrap();
let write = |name: &str, manifest: &str| {
let dir = workdir.path().join("tools").join(name);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(PACKED_MANIFEST_ENTRY), manifest).unwrap();
};
write(
"noter",
"name: noter\ninput_schema: '{\"properties\":{\"note\":{\"format\":\"murmur-opaque\"}}}'\n",
);
write(
"writer",
"name: writer\ninput_schema: '{\"properties\":{\"path\":{\"type\":\"string\"}}}'\n",
);
write("bare", "name: bare\nversion: 0.1.0\n");
let map = ToolAnnotationMap::from_workdir(workdir.path());
assert!(map.for_tool("noter").is_opaque(&[key("note")]));
assert!(
map.for_tool("writer").is_empty() && map.for_tool("bare").is_empty(),
"a tool that annotated nothing keeps the key-name heuristic"
);
assert!(ToolAnnotationMap::from_workdir(Path::new("/nowhere/at/all")).is_empty());
}
}