use std::collections::{BTreeMap, BTreeSet};
use std::io::Write;
use std::process::{Command, Stdio};
use std::sync::OnceLock;
use serde::Deserialize;
pub const CATALOG_SOURCE: &str = include_str!("../assets/departures.yml");
pub const EVIDENCE_DIR: &str = "docs/departures/evidence/";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum Kind {
FernDefect,
Branding,
Packaging,
Provenance,
Ordering,
RicherDocumentation,
}
impl Kind {
pub const ALL: [Kind; 6] = [
Kind::FernDefect,
Kind::Branding,
Kind::Packaging,
Kind::Provenance,
Kind::Ordering,
Kind::RicherDocumentation,
];
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Kind::FernDefect => "fern-defect",
Kind::Branding => "branding",
Kind::Packaging => "packaging",
Kind::Provenance => "provenance",
Kind::Ordering => "ordering",
Kind::RicherDocumentation => "richer-documentation",
}
}
#[must_use]
pub fn meaning(self) -> &'static str {
match self {
Kind::FernDefect => {
"Fern's output is wrong under the defect rule; crozier writes the correct output."
}
Kind::Branding => "crozier names itself where Fern names itself.",
Kind::Packaging => {
"crozier writes the packaged SDK's publishing details from its own settings."
}
Kind::Provenance => "crozier writes a fixed record of how the SDK was generated.",
Kind::Ordering => {
"crozier writes statements whose order has no effect in its own deterministic order."
}
Kind::RicherDocumentation => {
"Both outputs are valid; crozier's documents more of what the specification declares."
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Departure {
pub id: String,
pub kind: Kind,
pub trigger: String,
pub fern: String,
pub crozier: String,
pub reason: String,
pub evidence: String,
}
pub const KEYS: [&str; 7] = [
"id", "kind", "trigger", "fern", "crozier", "reason", "evidence",
];
pub fn parse(text: &str) -> Result<Vec<Departure>, Vec<String>> {
let entries: Vec<Departure> = crate::yaml::from_str(text).map_err(|error| {
vec![format!(
"the departure catalog is a list of entries holding exactly the keys {}: {error}",
KEYS.join(", ")
)]
})?;
let failures = validation_failures(&entries);
if failures.is_empty() {
Ok(entries)
} else {
Err(failures)
}
}
#[must_use]
pub fn catalog() -> &'static [Departure] {
static CATALOG: OnceLock<Vec<Departure>> = OnceLock::new();
CATALOG.get_or_init(|| {
parse(CATALOG_SOURCE).unwrap_or_else(|failures| {
panic!(
"the compiled departure catalog is invalid:\n{}",
failures.join("\n")
)
})
})
}
#[must_use]
pub fn find(id: &str) -> Option<&'static Departure> {
catalog().iter().find(|departure| departure.id == id)
}
fn is_slug(id: &str) -> bool {
!id.is_empty()
&& !id.starts_with('-')
&& !id.ends_with('-')
&& !id.contains("--")
&& id
.bytes()
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
}
#[must_use]
pub fn validation_failures(entries: &[Departure]) -> Vec<String> {
let mut failures = Vec::new();
for pair in entries.windows(2) {
let (previous, entry) = (&pair[0], &pair[1]);
if previous.id == entry.id {
failures.push(format!(
"departure `{}`: the id appears twice; ids are unique",
entry.id
));
} else if previous.id > entry.id {
failures.push(format!(
"departure `{}`: follows `{}`; entries are sorted by id",
entry.id, previous.id
));
}
}
for entry in entries {
let id = &entry.id;
if !is_slug(id) {
failures.push(format!(
"departure `{id}`: the id is not a lower-kebab slug"
));
}
for (key, value) in [
("trigger", &entry.trigger),
("fern", &entry.fern),
("crozier", &entry.crozier),
("reason", &entry.reason),
] {
if value.trim().is_empty() {
failures.push(format!("departure `{id}`: `{key}` is empty"));
}
}
let evidence = &entry.evidence;
let plain = !evidence.contains('\\')
&& evidence
.split('/')
.all(|part| !part.is_empty() && part != "." && part != "..");
if !plain || !evidence.starts_with(EVIDENCE_DIR) || !evidence.ends_with(".md") {
failures.push(format!(
"departure `{id}`: `evidence` `{evidence}` is not a note under {EVIDENCE_DIR}"
));
}
if rule(id).is_none() {
failures.push(format!(
"departure `{id}`: no rule of that id in src/departures.rs; every entry is \
applied through its rule"
));
}
}
let ids: BTreeSet<&str> = entries.iter().map(|entry| entry.id.as_str()).collect();
for id in RULE_IDS {
if !ids.contains(id) {
failures.push(format!(
"departure rule `{id}` has no catalog entry; a rule is applied only as an entry's"
));
}
}
failures
}
pub const REFERENCE_MARKERS: (&str, &str) = (
"<!-- BEGIN GENERATED CATALOG: edit assets/departures.yml, then regenerate -->",
"<!-- END GENERATED CATALOG -->",
);
#[must_use]
pub fn render_reference(entries: &[Departure]) -> String {
let one_line = |text: &str| text.split_whitespace().collect::<Vec<_>>().join(" ");
let mut out = format!("{}\n\n", REFERENCE_MARKERS.0);
out.push_str("| Kind | Entries | Meaning |\n| --- | --- | --- |\n");
for kind in Kind::ALL {
let count = entries.iter().filter(|entry| entry.kind == kind).count();
out.push_str(&format!(
"| `{}` | {count} | {} |\n",
kind.as_str(),
kind.meaning()
));
}
for entry in entries {
out.push_str(&format!(
"\n### `{}`\n\n- **Kind:** `{}`\n- **Trigger:** {}\n- **Fern writes:** {}\n\
- **crozier writes:** {}\n- **Why:** {}\n- **Evidence:** [`{}`](../../{})\n",
entry.id,
entry.kind.as_str(),
one_line(&entry.trigger),
one_line(&entry.fern),
one_line(&entry.crozier),
one_line(&entry.reason),
entry.evidence,
entry.evidence,
));
}
out.push_str(&format!("\n{}\n", REFERENCE_MARKERS.1));
out
}
#[derive(Debug, Default)]
pub struct Context {
source_document: Option<crate::openapi::OpenApi>,
source_enum_examples: OnceLock<Vec<SourceEnumExample>>,
source_int64_examples: OnceLock<BTreeSet<i64>>,
source_string_consts: OnceLock<BTreeSet<String>>,
roots: Option<(std::path::PathBuf, std::path::PathBuf)>,
reference_classes: OnceLock<BTreeSet<String>>,
crozier_classes: OnceLock<BTreeSet<String>>,
crozier_lifted: OnceLock<BTreeSet<String>>,
crozier_constant_headers: OnceLock<BTreeSet<(String, String)>>,
reference_nullable_items: OnceLock<BTreeSet<String>>,
crozier_project: OnceLock<Option<String>>,
crozier_returns: OnceLock<BTreeMap<(String, String), String>>,
request_examples: OnceLock<RequestExampleContext>,
empty_namespace: OnceLock<EmptyNamespace>,
crozier_paths: OnceLock<BTreeSet<String>>,
}
#[derive(Debug, Default)]
struct EmptyNamespace {
replaced: BTreeSet<String>,
reference_only: BTreeSet<String>,
crozier_only: BTreeSet<String>,
}
impl Context {
pub fn from_sources<'a>(
reference: impl IntoIterator<Item = (&'a str, &'a str)>,
crozier: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> Self {
let reference: Vec<(&str, &str)> = reference.into_iter().collect();
let crozier: Vec<(&str, &str)> = crozier.into_iter().collect();
let project = crozier
.iter()
.find(|(path, _)| *path == "pyproject.toml")
.and_then(|(_, text)| project_name(text));
Context {
source_document: None,
source_enum_examples: OnceLock::new(),
source_int64_examples: OnceLock::new(),
source_string_consts: OnceLock::new(),
roots: None,
reference_classes: OnceLock::from(classes(reference.iter().copied())),
crozier_classes: OnceLock::from(classes(crozier.iter().copied())),
crozier_lifted: OnceLock::from(lifted_parameters(crozier.iter().copied())),
crozier_constant_headers: OnceLock::from(constant_headers(crozier.iter().copied())),
reference_nullable_items: OnceLock::from(nullable_item_parameters(
reference
.iter()
.filter(|(path, _)| *path == "reference.md")
.map(|(_, text)| *text),
)),
crozier_project: OnceLock::from(project),
empty_namespace: OnceLock::from(EmptyNamespace::default()),
crozier_returns: OnceLock::from(method_returns(crozier.iter().copied())),
request_examples: OnceLock::from(request_example_context(crozier.iter().copied())),
crozier_paths: OnceLock::from(
crozier
.iter()
.map(|(path, _)| (*path).to_string())
.collect::<BTreeSet<_>>(),
),
}
}
#[must_use]
pub fn from_trees(reference: &std::path::Path, crozier: &std::path::Path) -> Self {
Context {
roots: Some((reference.to_path_buf(), crozier.to_path_buf())),
..Context::default()
}
}
#[must_use]
pub fn with_source_document(mut self, source: crate::openapi::OpenApi) -> Self {
self.source_document = Some(source);
self.source_enum_examples = OnceLock::new();
self.source_int64_examples = OnceLock::new();
self.source_string_consts = OnceLock::new();
self
}
pub fn reference_classes(&self) -> &BTreeSet<String> {
self.reference_classes
.get_or_init(|| self.tree_classes(|(reference, _)| reference))
}
pub fn crozier_classes(&self) -> &BTreeSet<String> {
self.crozier_classes
.get_or_init(|| self.tree_classes(|(_, crozier)| crozier))
}
pub fn crozier_lifted_parameters(&self) -> &BTreeSet<String> {
self.crozier_lifted.get_or_init(|| {
let Some((_, root)) = self.roots.as_ref() else {
return BTreeSet::new();
};
let sources: Vec<(String, String)> = python_sources(root);
lifted_parameters(
sources
.iter()
.map(|(rel, text)| (rel.as_str(), text.as_str())),
)
})
}
pub fn crozier_constant_headers(&self) -> &BTreeSet<(String, String)> {
self.crozier_constant_headers.get_or_init(|| {
let Some((_, root)) = self.roots.as_ref() else {
return BTreeSet::new();
};
let sources = python_sources(root);
constant_headers(
sources
.iter()
.map(|(rel, text)| (rel.as_str(), text.as_str())),
)
})
}
pub fn reference_nullable_items(&self) -> &BTreeSet<String> {
self.reference_nullable_items.get_or_init(|| {
let Some((root, _)) = self.roots.as_ref() else {
return BTreeSet::new();
};
let text = std::fs::read_to_string(root.join("reference.md")).unwrap_or_default();
nullable_item_parameters([text.as_str()])
})
}
pub fn crozier_method_returns(&self) -> &BTreeMap<(String, String), String> {
self.crozier_returns.get_or_init(|| {
let Some((_, root)) = self.roots.as_ref() else {
return BTreeMap::new();
};
let sources = python_sources(root);
method_returns(
sources
.iter()
.map(|(rel, text)| (rel.as_str(), text.as_str())),
)
})
}
pub fn crozier_project(&self) -> Option<&str> {
self.crozier_project
.get_or_init(|| {
let (_, crozier) = self.roots.as_ref()?;
project_name(&std::fs::read_to_string(crozier.join("pyproject.toml")).ok()?)
})
.as_deref()
}
fn crozier_paths(&self) -> &BTreeSet<String> {
self.crozier_paths.get_or_init(|| {
self.roots
.as_ref()
.and_then(|(_, crozier)| crate::parity::walk_files(crozier).ok())
.unwrap_or_default()
.into_iter()
.collect()
})
}
fn empty_namespace(&self) -> &EmptyNamespace {
self.empty_namespace.get_or_init(|| {
self.roots
.as_ref()
.map(|(reference, crozier)| empty_namespace_moves(reference, crozier))
.unwrap_or_default()
})
}
fn request_examples(&self) -> &RequestExampleContext {
self.request_examples.get_or_init(|| {
let Some((_, root)) = &self.roots else {
return RequestExampleContext::default();
};
let sources = python_sources(root);
request_example_context(
sources
.iter()
.map(|(rel, text)| (rel.as_str(), text.as_str())),
)
})
}
fn tree_classes(
&self,
side: impl Fn(&(std::path::PathBuf, std::path::PathBuf)) -> &std::path::PathBuf,
) -> BTreeSet<String> {
let Some(root) = self.roots.as_ref().map(side) else {
return BTreeSet::new();
};
let sources = python_sources(root);
classes(
sources
.iter()
.map(|(rel, text)| (rel.as_str(), text.as_str())),
)
}
}
#[derive(Debug)]
struct SourceEnumExample {
keyword: String,
enum_name: String,
first_value: String,
valid_value: String,
first_member: String,
valid_member: String,
}
fn source_enum_examples(source: Option<&crate::openapi::OpenApi>) -> Vec<SourceEnumExample> {
let Some(source) = source else {
return Vec::new();
};
let Ok(config) = crate::config::GenerateConfig::new(
std::path::PathBuf::new(),
std::path::PathBuf::new(),
Some("parity".into()),
Some("parity".into()),
Some("ParityClient".into()),
crate::settings::ExtraFields::Allow,
&source.info.title,
) else {
return Vec::new();
};
let ir = crate::ir::build(source, &config);
let enums: std::collections::HashMap<_, _> = ir
.types
.iter()
.filter_map(|decl| match decl {
crate::ir::TypeDecl::Enum(declaration) if declaration.has_corrected_example() => {
Some((declaration.name.as_str(), declaration))
}
_ => None,
})
.collect();
fn enum_name(reference: &crate::ir::TypeRef) -> Option<&str> {
match reference {
crate::ir::TypeRef::Named(name) => Some(name),
crate::ir::TypeRef::Optional(inner) => enum_name(inner),
_ => None,
}
}
let mut examples = Vec::new();
let mut add = |keyword: &str, reference: &crate::ir::TypeRef| {
let Some(declaration) = enum_name(reference).and_then(|name| enums.get(name)) else {
return;
};
let (Some(first), Some(valid)) =
(declaration.members.first(), declaration.example_member())
else {
return;
};
if first.value == valid.value {
return;
}
examples.push(SourceEnumExample {
keyword: keyword.into(),
enum_name: declaration.name.clone(),
first_value: first.value.clone(),
valid_value: valid.value.clone(),
first_member: first.name.clone(),
valid_member: valid.name.clone(),
});
};
for declaration in &ir.types {
if let crate::ir::TypeDecl::Object(object) = declaration {
for field in &object.fields {
add(&field.py_name, &field.type_ref);
}
}
}
for endpoint in &ir.endpoints {
if let Some(crate::ir::RequestBody::Inline(fields)) = &endpoint.request_body {
for field in fields {
add(&field.py_name, &field.type_ref);
}
}
}
examples
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct MethodKey {
raw_client_path: String,
method_name: String,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum ExampleSide {
Reference,
Corrected,
}
#[derive(Debug, Default)]
struct RequestExampleContext {
binary_json_methods: BTreeSet<MethodKey>,
alias_fields: std::collections::BTreeMap<String, BTreeSet<String>>,
method_parameters: std::collections::BTreeMap<
MethodKey,
std::collections::BTreeMap<String, GeneratedParameter>,
>,
json_object_methods: BTreeSet<MethodKey>,
wire_fields: std::collections::BTreeMap<MethodKey, std::collections::BTreeMap<String, String>>,
}
#[derive(Debug, Default)]
struct GeneratedParameter {
annotation: String,
description: Vec<String>,
}
fn generated_method_parameters(
text: &str,
) -> std::collections::BTreeMap<String, std::collections::BTreeMap<String, GeneratedParameter>> {
let mut out = std::collections::BTreeMap::new();
let mut method = None;
let mut signature = String::new();
let mut in_signature = false;
let mut in_doc = false;
let mut parameters = false;
let mut parameter = None;
for line in text.lines() {
if let Some(name) = python_method_name(line) {
method = Some(name.to_string());
signature.clear();
in_signature = true;
in_doc = false;
parameters = false;
parameter = None;
}
let Some(method) = method.as_ref() else {
continue;
};
if in_signature {
signature.push_str(line.trim());
if line.trim_end().ends_with(':') {
in_signature = false;
if let Some((open, close)) = signature.find('(').zip(signature.rfind(')')) {
if open <= close {
let call = format!("method{}", &signature[open..=close]);
if let Some((_, arguments)) = example_call_arguments(&call) {
let fields = out
.entry(method.clone())
.or_insert_with(std::collections::BTreeMap::new);
for argument in arguments {
if let Some((name, annotation)) = argument.trim().split_once(':') {
if python_identifier(name) {
fields.insert(
name.to_string(),
GeneratedParameter {
annotation: annotation
.trim()
.split(" = ")
.next()
.unwrap_or_default()
.to_string(),
description: Vec::new(),
},
);
}
}
}
}
}
}
}
continue;
}
if line.trim() == "\"\"\"" {
in_doc = !in_doc;
parameters = false;
continue;
}
if !in_doc {
continue;
}
if line == " Parameters" {
parameters = true;
continue;
}
if line == " Returns" || line == " Yields" {
parameters = false;
}
if !parameters {
continue;
}
if let Some((name, _)) = line
.strip_prefix(" ")
.and_then(|line| line.split_once(" : "))
{
if python_identifier(name) {
parameter = Some(name.to_string());
continue;
}
}
if let (Some(name), Some(description)) =
(parameter.as_ref(), line.strip_prefix(" "))
{
if let Some(field) = out.get_mut(method).and_then(|fields| fields.get_mut(name)) {
field.description.push(description.replace("\\\\", "\\"));
}
}
}
out
}
fn python_identifier(value: &str) -> bool {
let mut chars = value.chars();
chars.next().is_some_and(|c| c == '_' || c.is_alphabetic())
&& chars.all(|c| c == '_' || c.is_alphanumeric())
}
fn python_method_name(line: &str) -> Option<&str> {
line.strip_prefix(" def ")
.or_else(|| line.strip_prefix(" async def "))?
.split_once('(')
.map(|(name, _)| name)
.filter(|name| python_identifier(name))
}
fn request_example_context<'a>(
sources: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> RequestExampleContext {
let sources: Vec<_> = sources.into_iter().collect();
let aliases: std::collections::BTreeMap<_, _> = sources
.iter()
.filter(|(rel, _)| rel.ends_with(".py"))
.flat_map(|(_, text)| text.lines())
.filter_map(|line| line.split_once(" = "))
.filter(|(name, _)| python_identifier(name))
.collect();
let mut out = RequestExampleContext::default();
let mut model_fields: std::collections::BTreeMap<String, BTreeSet<String>> =
std::collections::BTreeMap::new();
for (_, text) in &sources {
let mut model = None;
for line in text.lines() {
if let Some(name) = line
.strip_prefix("class ")
.and_then(|line| line.split_once('(').map(|(name, _)| name))
{
model = Some(name.to_string());
}
if let (Some(model), Some((name, annotation))) = (
model.as_ref(),
line.strip_prefix(" ")
.and_then(|line| line.split_once(": ")),
) {
if python_identifier(name) && !annotation.starts_with("typing.ClassVar[") {
model_fields
.entry(model.clone())
.or_default()
.insert(name.to_string());
}
}
}
}
for (name, target) in &aliases {
let mut current = *target;
let mut seen = BTreeSet::new();
while seen.insert(current) {
if let Some(fields) = model_fields
.get(current)
.filter(|fields| !fields.is_empty())
{
out.alias_fields.insert((*name).to_string(), fields.clone());
break;
}
let Some(next) = aliases.get(current) else {
break;
};
current = next;
}
}
let mut bytes_methods = BTreeSet::new();
let mut json_methods = BTreeSet::new();
let mut json_headers = BTreeSet::new();
for (rel, text) in sources
.iter()
.filter(|(rel, _)| *rel == "raw_client.py" || rel.ends_with("/raw_client.py"))
{
for (method, fields) in generated_method_parameters(text) {
out.method_parameters.insert(
MethodKey {
raw_client_path: (*rel).to_string(),
method_name: method,
},
fields,
);
}
let mut method = None;
let mut signature = false;
for line in text.lines() {
if let Some(name) = python_method_name(line) {
method = Some(name);
signature = true;
}
let Some(method) = method else {
continue;
};
let key = MethodKey {
raw_client_path: (*rel).to_string(),
method_name: method.to_string(),
};
if signature
&& compact_python(&[line])
.split(',')
.any(|field| field == "request:bytes")
{
bytes_methods.insert(key.clone());
}
if line.trim_end().ends_with(':') {
signature = false;
}
if line.trim() == "json={" {
out.json_object_methods.insert(key.clone());
}
if line.trim() == "json=request," {
json_methods.insert(key.clone());
}
if line.trim() == "\"content-type\": \"application/json\"," {
json_headers.insert(key.clone());
}
let Some((wire, value)) = line
.trim()
.strip_prefix('"')
.and_then(|s| s.split_once("\": "))
else {
continue;
};
let value = value.trim_end_matches(',');
let variable = value
.split_once("file=")
.and_then(|(_, rest)| rest.split_once(',').map(|(name, _)| name))
.unwrap_or(value);
if python_identifier(variable) {
out.wire_fields
.entry(key)
.or_default()
.insert(variable.to_string(), wire.to_string());
}
}
}
out.binary_json_methods = bytes_methods
.into_iter()
.filter(|key| json_methods.contains(key) && json_headers.contains(key))
.collect();
out
}
fn example_call_arguments(call: &str) -> Option<(&str, Vec<&str>)> {
let (prefix, _) = call.split_once('(')?;
let open = prefix.len();
let mut depth = 0usize;
let mut quote = None;
let mut escaped = false;
let mut start = open + 1;
let mut arguments = Vec::new();
for (index, character) in call.char_indices().skip_while(|(index, _)| *index < open) {
if let Some(delimiter) = quote {
if escaped {
escaped = false;
} else if character == '\\' {
escaped = true;
} else if character == delimiter {
quote = None;
}
continue;
}
match character {
'\'' | '"' => quote = Some(character),
'(' | '[' | '{' => depth += 1,
')' | ']' | '}' => {
depth = depth.checked_sub(1)?;
if depth == 0 {
if character != ')' || index + 1 != call.len() {
return None;
}
if start < index {
arguments.push(&call[start..index]);
}
return Some((prefix, arguments));
}
}
',' if depth == 1 => {
if start < index {
arguments.push(&call[start..index]);
}
start = index + 1;
}
_ => {}
}
}
None
}
fn multipart_example_lines<'a>(
lines: &'a [&'a str],
raw_rel: &str,
context: &RequestExampleContext,
side: ExampleSide,
) -> Option<(Vec<String>, bool)> {
let mut kept = Vec::new();
let mut signature = Vec::new();
let mut method = "";
let mut in_doc = false;
let mut in_example = false;
let mut allowed = BTreeSet::new();
let mut removed = false;
let mut index = 0;
while index < lines.len() {
let line = lines[index];
if let Some(name) = python_method_name(line) {
method = name;
signature.clear();
in_doc = false;
in_example = false;
allowed.clear();
}
if line.trim() == "\"\"\"" {
if !in_doc {
let signature = compact_python(&signature);
let fields: Vec<_> = signature
.split(',')
.filter_map(|part| {
let (name, annotation) = part.split_once(':')?;
python_identifier(name).then_some((name, annotation))
})
.collect();
let key = MethodKey {
raw_client_path: raw_rel.to_string(),
method_name: method.to_string(),
};
if let Some(wires) = context.wire_fields.get(&key) {
for (name, annotation) in &fields {
if annotation.contains('=') || !annotation.contains("core.File") {
continue;
}
let Some(wire) = wires.get(*name) else {
continue;
};
let value =
if annotation.contains("Sequence[") || annotation.contains("List[") {
format!("[\"example_{wire}\"]")
} else {
format!("\"example_{wire}\"")
};
allowed.insert(format!("{name}={value}"));
}
}
}
in_doc = !in_doc;
in_example = false;
} else if !in_doc && allowed.is_empty() {
signature.push(line);
}
if in_doc && line.trim() == "Examples" {
in_example = true;
}
let call_head = line
.trim_start()
.strip_prefix("await ")
.unwrap_or(line.trim_start());
let is_method_call = call_head
.strip_prefix("client.")
.and_then(|s| s.split_once('('))
.is_some_and(|(name, _)| name.rsplit('.').next() == Some(method));
if in_doc && in_example && !allowed.is_empty() && is_method_call {
let mut end = index;
loop {
let compact = compact_python(&lines[index..=end]);
if let Some((prefix, arguments)) = example_call_arguments(&compact) {
let args: Vec<_> = arguments
.into_iter()
.filter(|argument| {
if side == ExampleSide::Corrected && allowed.contains(*argument) {
removed = true;
false
} else {
true
}
})
.collect();
kept.push(format!("{prefix}({})", args.join(",")));
index = end + 1;
break;
}
end += 1;
if end >= lines.len() || lines[end].trim() == "\"\"\"" {
return None;
}
}
} else {
kept.push(line.to_string());
index += 1;
}
}
Some((kept, removed))
}
fn multipart_object_required_file_example(pair: &Pair<'_>) -> Result<Option<Region>, String> {
let Some(prefix) = pair.rel.strip_suffix("client.py") else {
return Ok(None);
};
if !prefix.is_empty() && !prefix.ends_with('/') {
return Ok(None);
}
let raw_rel = format!("{prefix}raw_client.py");
let context = pair.context.request_examples();
let Some((reference, _)) =
multipart_example_lines(pair.fern, &raw_rel, context, ExampleSide::Reference)
else {
return Ok(None);
};
let Some((corrected, removed)) =
multipart_example_lines(pair.crozier, &raw_rel, context, ExampleSide::Corrected)
else {
return Ok(None);
};
Ok((removed && reference == corrected)
.then(|| differing_window(pair.fern, pair.crozier))
.flatten())
}
fn binary_json_example_lines(
lines: &[&str],
rel: &str,
context: &RequestExampleContext,
side: ExampleSide,
) -> Option<(Vec<String>, bool)> {
let reference = rel == "reference.md";
let readme = rel == "README.md";
let raw = rel
.strip_suffix("client.py")
.filter(|prefix| prefix.is_empty() || prefix.ends_with('/'))
.map(|prefix| format!("{prefix}raw_client.py"));
if !reference && !readme && raw.is_none() {
return None;
}
let mut method = None;
let mut doc = false;
let mut example = false;
let mut kept = Vec::new();
let mut changed = false;
let mut index = 0;
while index < lines.len() {
let line = lines[index];
if reference {
if let Some((_, after)) = line.split_once("<details><summary><code>client.") {
method = after
.split_once("<a href=\"")
.and_then(|(_, after)| after.split_once("\">"))
.and_then(|(path, after)| {
after.split_once("</a>").map(|(name, _)| (path, name))
})
.and_then(|(path, name)| {
path.strip_suffix("client.py").map(|prefix| MethodKey {
raw_client_path: format!("{prefix}raw_client.py"),
method_name: name.to_string(),
})
})
.filter(|key| context.binary_json_methods.contains(key));
}
} else {
if let Some(name) = python_method_name(line) {
method = raw
.as_ref()
.map(|raw| MethodKey {
raw_client_path: raw.clone(),
method_name: name.to_string(),
})
.filter(|key| context.binary_json_methods.contains(key));
doc = false;
example = false;
}
if line.trim() == "\"\"\"" {
doc = !doc;
example = false;
} else if doc && line.trim() == "Examples" {
example = true;
}
}
let head = line.trim().strip_prefix("await ").unwrap_or(line.trim());
if readme {
let qualified = head
.strip_prefix("client.")
.and_then(|head| head.split_once('('))
.map(|(name, _)| name);
method = qualified.and_then(|qualified| {
context
.binary_json_methods
.iter()
.find(|key| {
let prefix = key
.raw_client_path
.strip_suffix("raw_client.py")
.unwrap_or("")
.trim_end_matches('/');
let module = prefix
.strip_prefix("src/")
.map_or(prefix, |prefix| {
prefix.split_once('/').map_or("", |(_, module)| module)
})
.replace('/', ".");
let expected = if module.is_empty() {
key.method_name.clone()
} else {
format!("{module}.{}", key.method_name)
};
qualified == expected
})
.cloned()
});
}
let call_method = head
.strip_prefix("client.")
.and_then(|head| head.split_once('('))
.and_then(|(name, _)| name.rsplit('.').next());
if (reference || readme || doc && example)
&& method
.as_ref()
.is_some_and(|key| call_method == Some(key.method_name.as_str()))
{
let mut end = index;
loop {
let compact = compact_python(&lines[index..=end]);
if let Some((prefix, arguments)) = example_call_arguments(&compact) {
let arguments: Vec<_> = arguments
.into_iter()
.map(|argument| {
if side == ExampleSide::Corrected && argument == "request=b\"string\"" {
changed = true;
"request=\"string\""
} else {
argument
}
})
.collect();
kept.push(format!("{prefix}({})", arguments.join(",")));
index = end + 1;
break;
}
end += 1;
if end >= lines.len() || lines[end].trim() == "\"\"\"" {
return None;
}
}
} else {
kept.push(line.to_string());
index += 1;
}
}
Some((kept, changed))
}
fn binary_json_body_example(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != "reference.md"
&& pair.rel != "README.md"
&& !pair
.rel
.strip_suffix("client.py")
.is_some_and(|prefix| prefix.is_empty() || prefix.ends_with('/'))
{
return Ok(None);
}
let context = pair.context.request_examples();
let Some((reference, _)) =
binary_json_example_lines(pair.fern, pair.rel, context, ExampleSide::Reference)
else {
return Ok(None);
};
let Some((corrected, changed)) =
binary_json_example_lines(pair.crozier, pair.rel, context, ExampleSide::Corrected)
else {
return Ok(None);
};
Ok((changed && reference == corrected)
.then(|| differing_window(pair.fern, pair.crozier))
.flatten())
}
fn reference_method_key(line: &str) -> Option<MethodKey> {
let (_, after) = line.split_once("<details><summary><code>client.")?;
let (_, after) = after.split_once("<a href=\"")?;
let (path, after) = after.split_once("\">")?;
let (name, _) = after.split_once("</a>")?;
let prefix = path.strip_suffix("client.py")?;
Some(MethodKey {
raw_client_path: format!("{prefix}raw_client.py"),
method_name: name.to_string(),
})
}
fn reference_parameter_block<'a>(
lines: &'a [&'a str],
index: usize,
) -> Option<(&'a str, &'a str, usize, usize)> {
if lines.get(index..index + 3)? != ["<dl>", "<dd>", ""] {
return None;
}
let header = lines.get(index + 3)?.strip_prefix("**")?;
let (name, after) = header.split_once(":** `")?;
if !python_identifier(name) {
return None;
}
let (annotation, _) = after.split_once('`')?;
let close = (index + 4..lines.len()).find(|&i| lines[i] == "</dd>")?;
if lines.get(close + 1)? != &"</dl>" {
return None;
}
let end = close + 2 + usize::from(lines.get(close + 2) == Some(&""));
Some((name, annotation, close, end))
}
fn alias_reference_parameters(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != "reference.md" {
return Ok(None);
}
let context = pair.context.request_examples();
let mut method = None;
let mut targets = std::collections::BTreeMap::new();
let mut reference = Vec::new();
let mut index = 0;
while index < pair.fern.len() {
let line = pair.fern[index];
if let Some(key) = reference_method_key(line) {
method = Some(key);
}
if let Some((name, alias, close, end)) = reference_parameter_block(pair.fern, index) {
if name == "request" {
if let Some((key, fields, parameters, wires)) = method.as_ref().and_then(|key| {
Some((
key,
context.alias_fields.get(alias)?,
context.method_parameters.get(key)?,
context.wire_fields.get(key)?,
))
}) {
let plain = pair.fern[index + 3] == format!("**request:** `{alias}` ")
&& pair.fern[index + 4..close]
.iter()
.all(|line| line.trim().is_empty());
if plain
&& !parameters.contains_key("request")
&& context.json_object_methods.contains(key)
&& fields.iter().all(|field| {
parameters.contains_key(field) && wires.contains_key(field)
})
{
if targets.insert(key.clone(), fields.clone()).is_some() {
return Ok(None);
}
reference.push("<flattened-alias-parameters>".to_string());
index = end;
continue;
}
}
}
}
reference.push(line.to_string());
index += 1;
}
if targets.is_empty() {
return Ok(None);
}
let mut corrected = Vec::new();
let mut seen: std::collections::BTreeMap<MethodKey, BTreeSet<String>> =
std::collections::BTreeMap::new();
method = None;
index = 0;
while index < pair.crozier.len() {
let line = pair.crozier[index];
if let Some(key) = reference_method_key(line) {
method = Some(key);
}
if let Some((name, annotation, close, end)) = reference_parameter_block(pair.crozier, index)
{
if let Some((key, parameter)) = method.as_ref().and_then(|key| {
let fields = targets.get(key)?;
if !fields.contains(name) {
return None;
}
Some((key, context.method_parameters.get(key)?.get(name)?))
}) {
if annotation != crate::emit::reference_param_annotation(¶meter.annotation) {
return Ok(None);
}
let mut description = parameter.description.join("\n");
while description.ends_with('\n') {
description.pop();
}
let description = (!description.is_empty()).then_some(description.as_str());
let suffix = crate::emit::reference_param_suffix(description);
let expected = format!("**{name}:** `{annotation}`{suffix}\n ");
if pair.crozier[index + 3..close].join("\n") != expected {
return Ok(None);
}
let fields = seen.entry(key.clone()).or_default();
if !fields.insert(name.to_string()) {
return Ok(None);
}
if fields.len() == 1 {
corrected.push("<flattened-alias-parameters>".to_string());
}
index = end;
continue;
}
}
corrected.push(line.to_string());
index += 1;
}
if seen != targets {
return Ok(None);
}
Ok((reference == corrected)
.then(|| differing_window(pair.fern, pair.crozier))
.flatten())
}
fn python_sources(root: &std::path::Path) -> Vec<(String, String)> {
crate::parity::walk_files(root)
.unwrap_or_default()
.into_iter()
.filter(|rel| rel.ends_with(".py"))
.filter_map(|rel| {
let text = std::fs::read_to_string(root.join(&rel)).ok()?;
Some((rel, text))
})
.collect()
}
fn empty_namespace_moves(reference: &std::path::Path, crozier: &std::path::Path) -> EmptyNamespace {
let read = |root: &std::path::Path, rel: &str| {
std::fs::read_to_string(root.join(rel))
.ok()
.map(|text| crate::strip_python_comments(&text))
};
let leading_blank_free = |text: &str| text.trim_start_matches('\n').to_string();
let crozier_files = crate::parity::walk_files(crozier).unwrap_or_default();
let packages: BTreeSet<&str> = crozier_files
.iter()
.filter_map(|rel| rel.strip_suffix("/_/client.py"))
.filter(|pkg| {
pkg.strip_prefix("src/")
.is_some_and(|name| !name.contains('/'))
})
.collect();
let mut found = EmptyNamespace::default();
'packages: for pkg in packages {
let (Some(root_client), Some(moved_client)) = (
read(reference, &format!("{pkg}/client.py")),
read(crozier, &format!("{pkg}/_/client.py")),
) else {
continue;
};
if !root_client
.lines()
.any(|line| line.starts_with("from ..core."))
|| without_examples(&root_client) != without_examples(&moved_client)
{
continue;
}
let mut reference_only = BTreeSet::new();
let mut crozier_only = BTreeSet::new();
for rel in &crozier_files {
let Some(inner) = rel.strip_prefix(&format!("{pkg}/_/")) else {
continue;
};
let fern_rel = format!("{pkg}/{inner}");
match inner {
"client.py" | "types/__init__.py" => {}
"__init__.py" => {
let (ours, theirs) = (read(crozier, rel), read(reference, &fern_rel));
if ours.as_deref().map(leading_blank_free)
!= theirs.as_deref().map(leading_blank_free)
{
continue 'packages;
}
}
_ => {
let ours = read(crozier, rel);
if ours.is_none() || ours != read(reference, &fern_rel) {
continue 'packages;
}
if !crozier.join(&fern_rel).exists() {
reference_only.insert(fern_rel);
}
}
}
crozier_only.insert(rel.clone());
}
found.reference_only.extend(reference_only);
found.crozier_only.extend(crozier_only);
found
.replaced
.extend([format!("{pkg}/client.py"), format!("{pkg}/__init__.py")]);
}
found
}
fn without_examples(text: &str) -> Vec<&str> {
let lines: Vec<&str> = text.lines().collect();
let mut kept = Vec::with_capacity(lines.len());
let mut in_examples = false;
for (index, line) in lines.iter().enumerate() {
if in_examples {
in_examples = line.trim() != "\"\"\"";
if in_examples {
continue;
}
} else if line.trim() == "Examples"
&& lines
.get(index + 1)
.is_some_and(|next| next.trim() == "--------")
{
in_examples = true;
continue;
}
kept.push(*line);
}
kept
}
fn lifted_parameters<'a>(
sources: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> BTreeSet<String> {
const READ: &str = "encode_path_param(self._client_wrapper._";
sources
.into_iter()
.filter(|(path, _)| path.ends_with(".py"))
.flat_map(|(_, text)| {
text.match_indices(READ)
.map(move |(at, _)| &text[at + READ.len()..])
})
.filter_map(|rest| {
let name: String = rest
.chars()
.take_while(|c| c.is_alphanumeric() || *c == '_')
.collect();
(!name.is_empty() && rest[name.len()..].starts_with(')')).then_some(name)
})
.collect()
}
fn constant_headers<'a>(
sources: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> BTreeSet<(String, String)> {
let mut found = BTreeSet::new();
for (path, text) in sources {
if !path.ends_with("raw_client.py") {
continue;
}
let mut in_headers = false;
for line in text.lines() {
let trimmed = line.trim();
if trimmed == "headers={" {
in_headers = true;
continue;
}
if in_headers && trimmed.starts_with('}') {
in_headers = false;
continue;
}
if !in_headers {
continue;
}
let pair = trimmed
.strip_suffix(',')
.and_then(|entry| entry.split_once(": "))
.and_then(|(key, value)| {
let key = key.strip_prefix('"')?.strip_suffix('"')?;
let value = value.strip_prefix('"')?.strip_suffix('"')?;
(!key.contains('"') && !value.contains('"'))
.then(|| (key.to_string(), value.to_string()))
});
found.extend(pair);
}
}
found
}
fn nullable_items_types(item: &str) -> (String, String) {
(
format!(
"typing.Optional[typing.Union[typing.Optional[{item}], typing.Sequence[typing.Optional[{item}]]]]"
),
format!("typing.Optional[typing.Union[{item}, typing.Sequence[{item}]]]"),
)
}
fn documented_parameter(line: &str) -> Option<(&str, &str)> {
let (name, rest) = line.strip_prefix("**")?.split_once(":** `")?;
let (annotation, _) = rest.split_once('`')?;
Some((name, annotation))
}
fn nullable_item_parameters<'a>(texts: impl IntoIterator<Item = &'a str>) -> BTreeSet<String> {
texts
.into_iter()
.flat_map(str::lines)
.filter_map(documented_parameter)
.filter(|(_, annotation)| {
annotation
.strip_prefix("typing.Optional[typing.Union[typing.Optional[")
.and_then(|rest| rest.split_once("], "))
.is_some_and(|(item, _)| nullable_items_types(item).0 == *annotation)
})
.map(|(name, _)| name.to_string())
.collect()
}
fn project_name(pyproject: &str) -> Option<String> {
pyproject
.lines()
.find_map(|line| line.strip_prefix("name = \"")?.strip_suffix('"'))
.map(str::to_string)
}
fn classes<'a>(sources: impl IntoIterator<Item = (&'a str, &'a str)>) -> BTreeSet<String> {
sources
.into_iter()
.filter(|(path, _)| path.ends_with(".py"))
.flat_map(|(_, text)| text.lines())
.filter_map(|line| line.strip_prefix("class "))
.map(|rest| {
rest.chars()
.take_while(|c| c.is_alphanumeric() || *c == '_')
.collect::<String>()
})
.filter(|name| !name.is_empty())
.collect()
}
fn method_returns<'a>(
sources: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> BTreeMap<(String, String), String> {
let mut out = BTreeMap::new();
for (path, text) in sources {
if !path.ends_with(".py") {
continue;
}
let lines: Vec<&str> = text.lines().collect();
for (index, line) in lines.iter().enumerate() {
let Some(rest) = line
.strip_prefix(" def ")
.or_else(|| line.strip_prefix(" async def "))
else {
continue;
};
let name: String = rest
.chars()
.take_while(|c| c.is_alphanumeric() || *c == '_')
.collect();
let Some(end) = lines[index..]
.iter()
.position(|line| line.ends_with(':') && !line.starts_with(" "))
else {
continue;
};
let signature: String = lines[index..=index + end]
.iter()
.map(|line| line.trim())
.collect();
if let Some((_, annotation)) = signature.rsplit_once(") -> ") {
out.entry((path.to_string(), name))
.or_insert_with(|| annotation.trim_end_matches(':').to_string());
}
}
}
out
}
pub struct Pair<'a> {
pub rel: &'a str,
pub fern: &'a [&'a str],
pub crozier: &'a [&'a str],
pub context: &'a Context,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Region {
pub fern: std::ops::Range<usize>,
pub crozier: std::ops::Range<usize>,
}
pub type RegionRule = fn(&Pair<'_>) -> Result<Option<Region>, String>;
pub type LineRule = fn(&Pair<'_>, &str, &str) -> bool;
pub type AddedRule = fn(&Pair<'_>, &str) -> bool;
pub type AddedAtRule = fn(&Pair<'_>, usize) -> bool;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Side {
Reference,
Crozier,
}
pub type FileRule = fn(&Context, &str, Side) -> bool;
#[derive(Clone, Copy, Default)]
pub struct Rule {
pub region: Option<RegionRule>,
pub line: Option<LineRule>,
pub added: Option<AddedRule>,
pub added_at: Option<AddedAtRule>,
pub file: Option<FileRule>,
}
pub const RULE_IDS: [&str; 26] = [
"anyof-path-param-schema-example",
"binary-json-body-example",
"body-query-parameter-value",
"closed-empty-object-example",
"collapsed-anyof-property-description",
"constant-header-docs-arguments",
"date-header-constructor-example",
"docstring-literal-dollar-escape",
"emoji-tag-reference-identifiers",
"empty-namespace-package",
"fern-metadata-generator-config",
"init-type-checking-import-order",
"int64-example-rounded-out-of-range",
"lifted-base-path-docs-examples",
"multipart-object-required-file-example",
"nullable-items-docs",
"optional-lowered-required-query-array-example",
"pattern-narrowed-enum-example",
"readme-client-class-casing",
"repeated-credential-example-keyword",
"request-alias-reference-parameters",
"sdk-identity-header-prefix",
"sdk-name-version-headers",
"sdk-variable-docs-examples",
"stream-reference-return-type",
"text-response-optional-query-examples",
];
#[must_use]
pub fn rule(id: &str) -> Option<Rule> {
let none = Rule::default();
Some(match id {
"anyof-path-param-schema-example" => Rule {
line: Some(anyof_path_param_schema_example),
..none
},
"body-query-parameter-value" => Rule {
region: Some(body_query_parameter_value),
..none
},
"collapsed-anyof-property-description" => Rule {
line: Some(collapsed_anyof_property_description),
added_at: Some(collapsed_anyof_property_description_line),
..none
},
"emoji-tag-reference-identifiers" => Rule {
line: Some(emoji_tag_reference_identifiers),
..none
},
"optional-lowered-required-query-array-example" => Rule {
added_at: Some(optional_lowered_required_query_array_example),
..none
},
"text-response-optional-query-examples" => Rule {
added: Some(text_response_optional_query_examples),
..none
},
"closed-empty-object-example" => Rule {
region: Some(closed_empty_object_example_region),
line: Some(closed_empty_object_example_line),
..none
},
"constant-header-docs-arguments" => Rule {
region: Some(constant_header_docs_arguments),
..none
},
"date-header-constructor-example" => Rule {
region: Some(date_header_constructor_example),
..none
},
"docstring-literal-dollar-escape" => Rule {
line: Some(docstring_literal_dollar_escape),
..none
},
"int64-example-rounded-out-of-range" => Rule {
line: Some(int64_example_rounded_out_of_range),
..none
},
"empty-namespace-package" => Rule {
region: Some(empty_namespace_package_root),
file: Some(empty_namespace_package_file),
..none
},
"fern-metadata-generator-config" => Rule {
region: Some(metadata_generator_config),
..none
},
"binary-json-body-example" => Rule {
region: Some(binary_json_body_example),
..Rule::default()
},
"request-alias-reference-parameters" => Rule {
region: Some(alias_reference_parameters),
..none
},
"multipart-object-required-file-example" => Rule {
region: Some(multipart_object_required_file_example),
..none
},
"init-type-checking-import-order" => Rule {
region: Some(init_type_checking_import_order),
..none
},
"lifted-base-path-docs-examples" => Rule {
region: Some(lifted_base_path_docs_examples),
..none
},
"nullable-items-docs" => Rule {
region: Some(nullable_items_docs),
..none
},
"pattern-narrowed-enum-example" => Rule {
line: Some(pattern_narrowed_enum_example),
..none
},
"readme-client-class-casing" => Rule {
line: Some(readme_client_class_casing),
..none
},
"repeated-credential-example-keyword" => Rule {
region: Some(repeated_credential_example_keyword),
..none
},
"stream-reference-return-type" => Rule {
line: Some(stream_reference_return_type),
..none
},
"sdk-identity-header-prefix" => Rule {
line: Some(sdk_identity_header_prefix),
..none
},
"sdk-name-version-headers" => Rule {
line: Some(sdk_name_version_header),
added: Some(sdk_name_version_header_added),
..none
},
"sdk-variable-docs-examples" => Rule {
region: Some(sdk_variable_docs_examples),
..none
},
_ => return None,
})
}
fn inline_json_entries(lines: &[&str]) -> Vec<std::ops::Range<usize>> {
let mut entries = Vec::new();
let mut in_body = false;
let mut start = None;
for (index, line) in lines.iter().enumerate() {
if *line == " json={" {
in_body = true;
} else if in_body && *line == " }," {
if let Some(begin) = start.take() {
entries.push(begin..index);
}
in_body = false;
} else if in_body && line.starts_with(" \"") {
if let Some(begin) = start.replace(index) {
entries.push(begin..index);
}
}
}
entries
}
fn compact_python(lines: &[&str]) -> String {
let mut out = String::new();
let mut quote = None;
let mut escaped = false;
for character in lines.iter().flat_map(|line| line.chars()) {
if let Some(delimiter) = quote {
out.push(character);
if escaped {
escaped = false;
} else if character == '\\' {
escaped = true;
} else if character == delimiter {
quote = None;
}
} else if matches!(character, '\'' | '"') {
quote = Some(character);
out.push(character);
} else if !character.is_whitespace() {
out.push(character);
}
}
out
}
fn body_entry_value(expression: &str) -> Option<(&str, &str, bool)> {
let expression = expression.strip_prefix('"')?;
let (wire, value) = expression.split_once("\":")?;
let (value, converted) = value
.strip_prefix("convert_and_respect_annotation_metadata(object_=")
.map_or((value, false), |value| (value, true));
let end =
value.find(|character: char| !character.is_ascii_alphanumeric() && character != '_')?;
let name = &value[..end];
(!name.is_empty() && (converted || &value[end..] == ",")).then_some((wire, name, converted))
}
fn body_query_parameter_value(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != "raw_client.py" && !pair.rel.ends_with("/raw_client.py") {
return Ok(None);
}
let fern_entries = inline_json_entries(pair.fern);
let crozier_entries = inline_json_entries(pair.crozier);
if fern_entries.len() != crozier_entries.len() {
return Ok(None);
}
let mut corrected = Vec::new();
for (fern, crozier) in fern_entries.iter().zip(&crozier_entries) {
let reference = &pair.fern[fern.clone()];
let generated = &pair.crozier[crozier.clone()];
if reference == generated {
continue;
}
let a = compact_python(reference);
let b = compact_python(generated);
let Some((wire, query, converted)) = body_entry_value(&a) else {
return Ok(None);
};
let Some((body_wire, body, body_converted)) = body_entry_value(&b) else {
return Ok(None);
};
if wire != body_wire
|| converted != body_converted
|| body == query
|| !body.ends_with(&format!("_{query}"))
{
return Ok(None);
}
let method = pair.crozier[..crozier.start]
.iter()
.rposition(|line| line.starts_with(" def ") || line.starts_with(" async def "))
.unwrap_or(0);
let before = &pair.crozier[method..crozier.start];
let signature: Vec<_> = before
.iter()
.take_while(|line| line.trim_start() != "\"\"\"")
.collect();
let declared = |name: &str| {
signature.iter().any(|line| {
line.starts_with(&format!(" {name}:"))
|| ((line.starts_with(" def ") || line.starts_with(" async def "))
&& line.contains(&format!(", {name}:")))
})
};
let query_mapping = format!(" \"{wire}\": {query},");
if !declared(query) || !declared(body) || !before.contains(&query_mapping.as_str()) {
return Ok(None);
}
let replaced = if converted {
b.replacen(&format!("object_={body},"), &format!("object_={query},"), 1)
} else {
format!("\"{wire}\":{query},")
};
let normalize_call_comma = |value: &str| {
value
.strip_suffix(",),")
.map_or_else(|| value.to_owned(), |prefix| format!("{prefix}),"))
};
if normalize_call_comma(&replaced) != normalize_call_comma(&a) {
return Ok(None);
}
corrected.push((fern.clone(), crozier.clone()));
}
let Some((first_fern, first_crozier)) = corrected.first() else {
return Ok(None);
};
let (last_fern, last_crozier) = corrected.last().expect("nonempty corrections");
let fern_span = first_fern.start..last_fern.end;
let crozier_span = first_crozier.start..last_crozier.end;
let mut normalized = Vec::new();
let mut position = crozier_span.start;
for (fern, crozier) in &corrected {
normalized.extend_from_slice(&pair.crozier[position..crozier.start]);
normalized.extend_from_slice(&pair.fern[fern.clone()]);
position = crozier.end;
}
normalized.extend_from_slice(&pair.crozier[position..crozier_span.end]);
Ok(
(normalized == pair.fern[fern_span.clone()]).then_some(Region {
fern: fern_span,
crozier: crozier_span,
}),
)
}
const KEY_VALUE_PLACEHOLDER: &str = "\"key\": \"value\"";
fn keyword_argument<'l>(line: &'l str, value: &str) -> Option<(&'l str, &'l str)> {
let indent = &line[..line.len() - line.trim_start().len()];
let (keyword, rest) = line.trim_start().split_once('=')?;
let identifier = keyword
.chars()
.next()
.is_some_and(|first| first.is_ascii_alphabetic() || first == '_')
&& keyword
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || ch == '_');
(identifier && rest == value).then_some((indent, keyword))
}
fn closed_empty_object_example_block(fern: &[&str], crozier: &str) -> bool {
let [open, member, close] = fern else {
return false;
};
let Some((indent, keyword)) = keyword_argument(open, "{") else {
return false;
};
*member == format!("{indent} {KEY_VALUE_PLACEHOLDER}")
&& *close == format!("{indent}}},")
&& crozier == format!("{indent}{keyword}={{}},")
}
fn closed_empty_object_example_region(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !DOCS_FILES.contains(&pair.rel) {
return Ok(None);
}
let fern = pair.fern;
let mut collapsed: Vec<String> = Vec::new();
let mut span: Option<(usize, usize, usize)> = None;
let mut index = 0;
while index < fern.len() {
let line = fern
.get(index..index + 3)
.and_then(|block| Some((block, keyword_argument(block[0], "{")?)))
.map(|(block, (indent, keyword))| (block, format!("{indent}{keyword}={{}},")))
.filter(|(block, line)| closed_empty_object_example_block(block, line));
if let Some((_, line)) = line {
let begin = span.map_or(index, |(begin, _, _)| begin);
collapsed.push(line);
index += 3;
span = Some((begin, index, collapsed.len()));
} else {
collapsed.push(fern[index].to_string());
index += 1;
}
}
let Some((begin, fern_end, end)) = span else {
return Ok(None);
};
let equal = pair.crozier.get(begin..end).is_some_and(|region| {
region
.iter()
.zip(&collapsed[begin..end])
.all(|(crozier, collapsed)| crozier == collapsed)
});
Ok(equal.then_some(Region {
fern: begin..fern_end,
crozier: begin..end,
}))
}
fn closed_empty_object_example_line(_: &Pair<'_>, fern: &str, crozier: &str) -> bool {
keyword_argument(fern, &format!("{{{KEY_VALUE_PLACEHOLDER}}},"))
.is_some_and(|(indent, keyword)| crozier == format!("{indent}{keyword}={{}},"))
}
fn documented_example_line(rel: &str, lines: &[&str], candidate: &str) -> bool {
let mut in_doc = false;
let mut example = false;
let mut found = false;
for line in lines {
if rel.ends_with(".py") {
if line.matches("\"\"\"").count() % 2 == 1 {
in_doc = !in_doc;
example = false;
}
if in_doc && line.trim() == "Examples" {
example = true;
}
} else if line.trim() == "```python" {
example = true;
} else if line.trim() == "```" {
example = false;
}
if *line == candidate {
if !example {
return false;
}
found = true;
}
}
found
}
fn int64_example_rounded_out_of_range(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if pair.rel != "README.md" && pair.rel != "reference.md" && !high_level_client(pair.rel) {
return false;
}
let Some((prefix, fern_digits, crozier_digits)) = differing_integer(fern, crozier) else {
return false;
};
let (negative, unsigned) = prefix
.strip_suffix('-')
.map_or((false, prefix), |rest| (true, rest));
let after = &fern[prefix.len() + fern_digits.len()..];
let stands_alone =
(unsigned.ends_with(['=', '[']) || unsigned.ends_with(", ") || unsigned.trim().is_empty())
&& (after.is_empty() || after.starts_with([',', ']', ')']));
if !stands_alone {
return false;
}
let signed = |digits: &str| {
let magnitude = digits.parse::<i128>().ok()?;
Some(if negative { -magnitude } else { magnitude })
};
let in_int64 = |value: i128| i128::from(i64::MIN) <= value && value <= i128::from(i64::MAX);
let (Some(fern_value), Some(crozier_value)) = (signed(fern_digits), signed(crozier_digits))
else {
return false;
};
let Ok(exact) = i64::try_from(crozier_value) else {
return false;
};
let declared = pair.context.source_int64_examples.get_or_init(|| {
let mut examples = BTreeSet::new();
visit_source_schemas(pair.context.source_document.as_ref(), &mut |schema| {
if schema.format.as_deref() == Some("int64")
&& schema
.ty
.as_ref()
.and_then(crate::openapi::TypeField::primary)
== Some("integer")
{
examples.extend(
schema
.example
.iter()
.chain(&schema.examples)
.filter_map(serde_json::Value::as_i64),
);
}
});
examples
});
if !declared.contains(&exact) {
return false;
}
let rounded = crate::ir::js_integer_literal(&serde_json::Number::from(exact));
rounded.trim_start_matches('-') == fern_digits
&& !in_int64(fern_value)
&& documented_example_line(pair.rel, pair.fern, fern)
&& documented_example_line(pair.rel, pair.crozier, crozier)
}
fn visit_source_schemas(
source: Option<&crate::openapi::OpenApi>,
visit: &mut dyn FnMut(&crate::openapi::Schema),
) {
fn walk(schema: &crate::openapi::Schema, visit: &mut dyn FnMut(&crate::openapi::Schema)) {
visit(schema);
for property in schema.properties.values() {
walk(property, visit);
}
if let Some(items) = &schema.items {
walk(items, visit);
}
if let Some(crate::openapi::AdditionalProperties::Schema(value)) =
&schema.additional_properties
{
walk(value, visit);
}
for members in [&schema.one_of, &schema.any_of, &schema.all_of] {
for member in members.iter().flatten() {
walk(member, visit);
}
}
}
fn walk_content(
content: &indexmap::IndexMap<String, crate::openapi::MediaType>,
visit: &mut dyn FnMut(&crate::openapi::Schema),
) {
for schema in content.values().filter_map(|media| media.schema.as_ref()) {
walk(schema, visit);
}
}
let Some(source) = source else {
return;
};
for schema in source.components.schemas.values() {
walk(schema, visit);
}
let parameters = source
.components
.parameters
.values()
.chain(source.paths.values().flat_map(|item| {
item.parameters.iter().chain(
item.operations()
.into_iter()
.flat_map(|(_, op)| &op.parameters),
)
}));
for parameter in parameters {
if let Some(schema) = ¶meter.schema {
walk(schema, visit);
}
walk_content(¶meter.content, visit);
}
for body in source.components.request_bodies.values() {
walk_content(&body.content, visit);
}
for response in source.components.responses.values() {
walk_content(&response.content, visit);
}
for item in source.paths.values() {
for (_, operation) in item.operations() {
if let Some(body) = &operation.request_body {
walk_content(&body.content, visit);
}
for response in operation.responses.values() {
walk_content(&response.content, visit);
}
}
}
}
fn high_level_client(rel: &str) -> bool {
rel.rsplit('/').next() == Some("client.py")
}
fn differing_integer<'a>(fern: &'a str, crozier: &'a str) -> Option<(&'a str, &'a str, &'a str)> {
let (left, right) = (fern.as_bytes(), crozier.as_bytes());
let common_prefix = left
.iter()
.zip(right)
.take_while(|(left, right)| left == right)
.count();
let common_suffix = left
.iter()
.rev()
.zip(right.iter().rev())
.take_while(|(left, right)| left == right)
.count()
.min(left.len().min(right.len()) - common_prefix);
let leading_digits = left[..common_prefix]
.iter()
.rev()
.take_while(|byte| byte.is_ascii_digit())
.count();
let start = common_prefix - leading_digits;
let trailing_digits = left[left.len() - common_suffix..]
.iter()
.take_while(|byte| byte.is_ascii_digit())
.count();
let suffix = common_suffix - trailing_digits;
let fern_digits = fern.get(start..fern.len() - suffix)?;
let crozier_digits = crozier.get(start..crozier.len() - suffix)?;
let digits = |text: &str| !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit());
(fern_digits != crozier_digits && digits(fern_digits) && digits(crozier_digits))
.then(|| {
fern.get(..start)
.map(|prefix| (prefix, fern_digits, crozier_digits))
})
.flatten()
}
fn docstring_literal_dollar_escape(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if !high_level_client(pair.rel) || !crozier.contains('$') {
return false;
}
let consts = pair.context.source_string_consts.get_or_init(|| {
let mut consts = BTreeSet::new();
visit_source_schemas(pair.context.source_document.as_ref(), &mut |schema| {
if let Some(serde_json::Value::String(value)) = &schema.const_value {
consts.insert(value.clone());
}
});
consts
});
let keyword_const = crozier
.trim_start()
.split_once('=')
.is_some_and(|(name, value)| {
python_identifier(name)
&& value
.strip_suffix(',')
.and_then(|literal| serde_json::from_str::<String>(literal).ok())
.is_some_and(|literal| consts.contains(&literal))
});
keyword_const
&& fern == crozier.replace('$', "\\\\$")
&& documented_example_line(pair.rel, pair.fern, fern)
&& documented_example_line(pair.rel, pair.crozier, crozier)
}
fn pattern_narrowed_enum_example(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if pair.rel != "README.md" && pair.rel != "reference.md" && !pair.rel.ends_with("client.py") {
return false;
}
let examples = pair
.context
.source_enum_examples
.get_or_init(|| source_enum_examples(pair.context.source_document.as_ref()));
for example in examples {
for (old, new) in [
(
format!(
"{}={}.{},",
example.keyword, example.enum_name, example.first_member
),
format!(
"{}={}.{},",
example.keyword, example.enum_name, example.valid_member
),
),
(
format!(
"{}={},",
example.keyword,
serde_json::to_string(&example.first_value).unwrap_or_default()
),
format!(
"{}={},",
example.keyword,
serde_json::to_string(&example.valid_value).unwrap_or_default()
),
),
] {
let indent = fern.len() - fern.trim_start().len();
if fern[indent..] == old && crozier == format!("{}{new}", &fern[..indent]) {
return documented_example_line(pair.rel, pair.fern, fern)
&& documented_example_line(pair.rel, pair.crozier, crozier);
}
}
}
false
}
pub const FERN_METADATA: &str = ".fern/metadata.json";
fn metadata_generator_config(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != FERN_METADATA {
return Ok(None);
}
let (fern_text, crozier_text) = (pair.fern.join("\n"), pair.crozier.join("\n"));
if crozier_text != crate::emit::FERN_METADATA_RECORD {
return Ok(None);
}
let (Ok(serde_json::Value::Object(mut fern)), Ok(serde_json::Value::Object(mut crozier))) = (
serde_json::from_str::<serde_json::Value>(&fern_text),
serde_json::from_str::<serde_json::Value>(&crozier_text),
) else {
return Ok(None);
};
let (fern_config, crozier_config) = (
fern.remove("generatorConfig"),
crozier.remove("generatorConfig"),
);
if fern != crozier || fern_config == crozier_config {
return Ok(None);
}
let (Some(window), Member::Lines(crozier_block)) = (
differing_window(pair.fern, pair.crozier),
generator_config_member(&crozier_text),
) else {
return Ok(None);
};
let within = |range: &std::ops::Range<usize>, block: &std::ops::Range<usize>| {
range.is_empty() || (range.start >= block.start && range.end <= block.end)
};
let holds = match generator_config_member(&fern_text) {
Member::Lines(fern_block) => {
within(&window.fern, &fern_block) && within(&window.crozier, &crozier_block)
}
Member::Absent => window.fern.is_empty() && within(&window.crozier, &crozier_block),
Member::Shared => false,
};
Ok(holds.then_some(window))
}
#[derive(Debug, PartialEq, Eq)]
enum Member {
Absent,
Lines(std::ops::Range<usize>),
Shared,
}
fn string_end(bytes: &[u8], start: usize) -> Option<usize> {
let mut index = start + 1;
while index < bytes.len() {
match bytes[index] {
b'\\' => index += 2,
b'"' => return Some(index + 1),
_ => index += 1,
}
}
None
}
fn generator_config_member(text: &str) -> Member {
let bytes = text.as_bytes();
let (mut depth, mut index, mut found) = (0usize, 0usize, None);
while index < bytes.len() {
match bytes[index] {
b'"' => {
let Some(end) = string_end(bytes, index) else {
return Member::Absent;
};
let after = text[end..].trim_start();
if depth == 1
&& &text[index..end] == "\"generatorConfig\""
&& after.starts_with(':')
{
found = Some((index, text.len() - after.len() + 1));
break;
}
index = end;
continue;
}
b'{' | b'[' => depth += 1,
b'}' | b']' => depth = depth.saturating_sub(1),
_ => {}
}
index += 1;
}
let Some((key, value_start)) = found else {
return Member::Absent;
};
let (mut depth, mut index, mut value_end) = (0usize, value_start, None);
while index < bytes.len() {
match bytes[index] {
b'"' => {
let Some(end) = string_end(bytes, index) else {
return Member::Absent;
};
if depth == 0 {
value_end = Some(end);
}
index = end;
continue;
}
b'{' | b'[' => depth += 1,
b'}' | b']' if depth == 0 => break,
b'}' | b']' => {
depth -= 1;
if depth == 0 {
value_end = Some(index + 1);
break;
}
}
b',' if depth == 0 => break,
byte if depth == 0 && !byte.is_ascii_whitespace() => value_end = Some(index + 1),
_ => {}
}
index += 1;
}
let Some(value_end) = value_end else {
return Member::Absent;
};
let line_of = |offset: usize| text[..offset].matches('\n').count();
let line_start = text[..key].rfind('\n').map_or(0, |newline| newline + 1);
let line_end = text[value_end..]
.find('\n')
.map_or(text.len(), |newline| value_end + newline);
let owns = text[line_start..key].trim().is_empty()
&& matches!(text[value_end..line_end].trim(), "" | ",");
if owns {
Member::Lines(line_of(key)..line_of(value_end) + 1)
} else {
Member::Shared
}
}
#[must_use]
pub fn differing_window(fern: &[&str], crozier: &[&str]) -> Option<Region> {
if fern == crozier {
return None;
}
let prefix = fern
.iter()
.zip(crozier)
.take_while(|(left, right)| left == right)
.count();
let suffix = fern[prefix..]
.iter()
.rev()
.zip(crozier[prefix..].iter().rev())
.take_while(|(left, right)| left == right)
.count();
Some(Region {
fern: prefix..fern.len() - suffix,
crozier: prefix..crozier.len() - suffix,
})
}
fn init_type_checking_import_order(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != "__init__.py" && !pair.rel.ends_with("/__init__.py") {
return Ok(None);
}
let (Some(fern_block), Some(crozier_block)) = (
type_checking_block(pair.fern),
type_checking_block(pair.crozier),
) else {
return Ok(None);
};
let fern = &pair.fern[fern_block.clone()];
let crozier = &pair.crozier[crozier_block.clone()];
let Some(window) = differing_window(fern, crozier) else {
return Ok(None);
};
let (Some(fern_body), Some(crozier_body)) = (dedent(fern), dedent(crozier)) else {
return Ok(None);
};
let window = if ruff_isort(&fern_body)? == ruff_isort(&crozier_body)? {
window
} else {
match reordered_statements(fern, crozier) {
Some(window) => window,
None => return Ok(None),
}
};
Ok(Some(Region {
fern: fern_block.start + window.fern.start..fern_block.start + window.fern.end,
crozier: crozier_block.start + window.crozier.start
..crozier_block.start + window.crozier.end,
}))
}
fn reordered_statements(fern: &[&str], crozier: &[&str]) -> Option<Region> {
let (fern_statements, crozier_statements) = (statements(fern)?, statements(crozier)?);
let module = |side: &[&str], range: &std::ops::Range<usize>| -> String {
let head = side[range.start].trim_start();
head.split_once(" import")
.map_or(head, |(module, _)| module)
.to_string()
};
let fern_modules: Vec<String> = fern_statements
.iter()
.map(|range| module(fern, range))
.collect();
let crozier_modules: Vec<String> = crozier_statements
.iter()
.map(|range| module(crozier, range))
.collect();
let mut sorted = (fern_modules.clone(), crozier_modules.clone());
sorted.0.sort();
sorted.1.sort();
if sorted.0 != sorted.1 || sorted.0.windows(2).any(|pair| pair[0] == pair[1]) {
return None;
}
let first = fern_modules
.iter()
.zip(&crozier_modules)
.position(|(f, c)| f != c)?;
let last = fern_modules.len()
- 1
- fern_modules
.iter()
.rev()
.zip(crozier_modules.iter().rev())
.position(|(f, c)| f != c)?;
let settled = |modules: &[String]| -> Vec<String> {
modules[first..=last]
.iter()
.filter(|module| !matches!(module.as_str(), "from .client" | "from .environment"))
.cloned()
.collect()
};
if settled(&fern_modules) != settled(&crozier_modules) {
return None;
}
let text = |side: &[&str], range: &std::ops::Range<usize>| side[range.clone()].join("\n");
for (index, name) in fern_modules.iter().enumerate().take(last + 1).skip(first) {
let other = crozier_modules
.iter()
.position(|candidate| candidate == name)?;
if !(first..=last).contains(&other)
|| text(fern, &fern_statements[index]) != text(crozier, &crozier_statements[other])
{
return None;
}
}
Some(Region {
fern: fern_statements[first].start..fern_statements[last].end,
crozier: crozier_statements[first].start..crozier_statements[last].end,
})
}
fn statements(block: &[&str]) -> Option<Vec<std::ops::Range<usize>>> {
let starts: Vec<usize> = block
.iter()
.enumerate()
.filter(|(_, line)| line.starts_with(" from ") || line.starts_with(" import "))
.map(|(index, _)| index)
.collect();
if block[..starts.first().copied().unwrap_or(block.len())]
.iter()
.any(|line| !line.is_empty())
{
return None;
}
Some(
starts
.iter()
.enumerate()
.map(|(index, start)| *start..starts.get(index + 1).copied().unwrap_or(block.len()))
.collect(),
)
}
fn type_checking_block(lines: &[&str]) -> Option<std::ops::Range<usize>> {
let start = lines
.iter()
.position(|line| *line == "if typing.TYPE_CHECKING:")?
+ 1;
let end = lines[start..]
.iter()
.position(|line| !line.is_empty() && !line.starts_with(' '))
.map_or(lines.len(), |offset| start + offset);
Some(start..end)
}
fn dedent(lines: &[&str]) -> Option<String> {
let mut out = String::new();
for line in lines {
if !line.is_empty() {
out.push_str(line.strip_prefix(" ")?);
}
out.push('\n');
}
Some(out)
}
pub fn ruff_isort(source: &str) -> Result<String, String> {
let mut child = Command::new("ruff")
.args([
"check",
"--select",
"I",
"--fix",
"--stdin-filename",
"x.py",
"-",
])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|error| format!("could not run ruff (see docs/matching.md): {error}"))?;
child
.stdin
.take()
.ok_or_else(|| "ruff stdin was not piped".to_string())?
.write_all(source.as_bytes())
.map_err(|error| format!("could not write to ruff: {error}"))?;
let out = child
.wait_with_output()
.map_err(|error| format!("could not wait for ruff: {error}"))?;
if !out.status.success() {
return Err(format!(
"ruff isort failed ({}): {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
));
}
String::from_utf8(out.stdout).map_err(|error| format!("ruff output is not UTF-8: {error}"))
}
const DOCS_FILES: [&str; 2] = ["README.md", "reference.md"];
fn tokens(line: &str) -> Vec<&str> {
let mut out = Vec::new();
let mut start = None;
for (index, ch) in line.char_indices() {
let word = ch.is_alphanumeric() || ch == '_';
match (word, start) {
(true, None) => start = Some(index),
(false, Some(begun)) => {
out.push(&line[begun..index]);
out.push(&line[index..index + ch.len_utf8()]);
start = None;
}
(false, None) => out.push(&line[index..index + ch.len_utf8()]),
(true, Some(_)) => {}
}
}
if let Some(begun) = start {
out.push(&line[begun..]);
}
out
}
fn readme_client_class_casing(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if !DOCS_FILES.contains(&pair.rel) {
return false;
}
let (fern, crozier) = (tokens(fern), tokens(crozier));
if fern.len() != crozier.len() {
return false;
}
let mut renamed = 0;
for (fern, crozier) in fern.iter().zip(&crozier) {
if fern == crozier {
continue;
}
if !fern.eq_ignore_ascii_case(crozier)
|| !pair.context.crozier_classes().contains(*crozier)
|| pair.context.reference_classes().contains(*fern)
|| pair.context.crozier_classes().contains(*fern)
{
return false;
}
renamed += 1;
}
renamed > 0
}
fn stream_reference_return_type(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
const OPEN: &str = "<details><summary><code>client.";
const CLOSE: &str = "</code></summary>";
const FERN: &str = " -> typing.Iterator[bytes]";
if pair.rel != "reference.md" || !fern.starts_with(OPEN) {
return false;
}
let Some(head) = fern
.strip_suffix(CLOSE)
.and_then(|line| line.strip_suffix(FERN))
else {
return false;
};
let Some(annotation) = crozier
.strip_suffix(CLOSE)
.and_then(|line| line.strip_prefix(head))
.and_then(|rest| rest.strip_prefix(" -> "))
else {
return false;
};
if !annotation.starts_with("typing.Iterator[") || annotation == "typing.Iterator[bytes]" {
return false;
}
let Some((path, rest)) = head
.split_once("<a href=\"")
.and_then(|(_, rest)| rest.split_once("\">"))
else {
return false;
};
let Some((method, _)) = rest.split_once("</a>") else {
return false;
};
let returns = pair.context.crozier_method_returns();
let declared = |path: &str| returns.get(&(path.to_string(), method.to_string()));
declared(path)
.or_else(|| {
path.strip_prefix("src/")
.and_then(|packaged| packaged.split_once('/'))
.and_then(|(_, flat)| declared(flat))
})
.is_some_and(|declared| declared.replace("dt.", "datetime.") == annotation)
}
fn lifted_argument<'l>(line: &'l str, names: &BTreeSet<String>) -> Option<&'l str> {
let trimmed = line.trim_start();
if trimmed.len() == line.len() || !line.ends_with(',') {
return None;
}
let (name, _) = trimmed.split_once('=')?;
names.contains(name).then_some(name)
}
fn repeated_credential_example_keyword(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !DOCS_FILES.contains(&pair.rel)
&& pair.rel != "client.py"
&& !pair.rel.ends_with("/client.py")
{
return Ok(None);
}
let mut kept: Vec<&str> = Vec::with_capacity(pair.fern.len());
let mut keywords: Option<(usize, BTreeSet<&str>)> = None;
let mut removed = false;
for line in pair.fern {
let trimmed = line.trim_start();
let indent = line.len() - trimmed.len();
if opens_constructor_call(trimmed) {
keywords = Some((indent, BTreeSet::new()));
} else if let Some((opened, seen)) = keywords.as_mut() {
if trimmed == ")" && indent == *opened {
keywords = None;
} else if let Some((name, _)) = trimmed.split_once('=').filter(|_| line.ends_with(','))
{
if !seen.insert(name) {
removed = true;
continue;
}
}
}
kept.push(line);
}
if !removed || kept != pair.crozier {
return Ok(None);
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn empty_namespace_package_root(pair: &Pair<'_>) -> Result<Option<Region>, String> {
Ok(pair
.context
.empty_namespace()
.replaced
.contains(pair.rel)
.then_some(Region {
fern: 0..pair.fern.len(),
crozier: 0..pair.crozier.len(),
}))
}
fn empty_namespace_package_file(context: &Context, rel: &str, side: Side) -> bool {
let moves = context.empty_namespace();
match side {
Side::Reference => moves.reference_only.contains(rel),
Side::Crozier => moves.crozier_only.contains(rel),
}
}
fn call_opener<'l>(lines: &[&'l str], index: usize) -> Option<&'l str> {
lines[..index]
.iter()
.rev()
.find(|line| line.ends_with('('))
.map(|line| line.trim_start())
}
fn opens_method_call(opener: &str) -> bool {
let call = opener.strip_prefix("await ").unwrap_or(opener);
call.starts_with("client.")
}
fn opens_constructor_call(opener: &str) -> bool {
opener.strip_prefix("client = ").is_some_and(|class| {
class.strip_suffix('(').is_some_and(|name| {
!name.is_empty() && name.chars().all(|c| c.is_alphanumeric() || c == '_')
})
})
}
fn lifted_parameter_block(lines: &[&str], at: usize, names: &BTreeSet<String>) -> Option<usize> {
parameter_block(lines, at, |line| {
line.strip_prefix("**")
.and_then(|rest| rest.split_once(":** "))
.is_some_and(|(name, _)| names.contains(name))
})
}
fn parameter_block(lines: &[&str], at: usize, documents: impl Fn(&str) -> bool) -> Option<usize> {
let block = lines.get(at..at + 8)?;
let documented = documents(block[3]);
(block[0] == "<dl>"
&& block[1] == "<dd>"
&& block[2].is_empty()
&& documented
&& block[4].trim().is_empty()
&& block[5] == "</dd>"
&& block[6] == "</dl>"
&& block[7].is_empty())
.then_some(8)
}
fn fern_docs_without_lifted<'l>(
lines: &[&'l str],
names: &BTreeSet<String>,
) -> Option<Vec<&'l str>> {
let mut kept = Vec::with_capacity(lines.len());
let mut index = 0;
while index < lines.len() {
if let Some(length) = lifted_parameter_block(lines, index, names) {
index += length;
continue;
}
let in_method_call = lifted_argument(lines[index], names).is_some()
&& call_opener(lines, index).is_some_and(opens_method_call);
if !in_method_call {
kept.push(lines[index]);
}
index += 1;
}
(kept.len() < lines.len()).then_some(kept)
}
fn crozier_docs_without_lifted(lines: &[&str], names: &BTreeSet<String>) -> Option<Vec<String>> {
let mut kept: Vec<String> = Vec::with_capacity(lines.len());
let mut removed = false;
for (index, line) in lines.iter().enumerate() {
if lifted_argument(line, names).is_some()
&& call_opener(lines, index).is_some_and(opens_constructor_call)
{
removed = true;
continue;
}
let closes_empty = *line == ")"
&& kept
.last()
.is_some_and(|opener| opens_constructor_call(opener));
match kept.last_mut() {
Some(opener) if closes_empty => opener.push(')'),
_ => kept.push((*line).to_string()),
}
}
removed.then_some(kept)
}
fn lifted_base_path_docs_examples(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !DOCS_FILES.contains(&pair.rel) {
return Ok(None);
}
let names = pair.context.crozier_lifted_parameters();
if names.is_empty() {
return Ok(None);
}
if !pair.fern.iter().enumerate().any(|(index, line)| {
lifted_argument(line, names).is_some()
&& call_opener(pair.fern, index).is_some_and(opens_method_call)
}) {
return Ok(None);
}
let (Some(fern), Some(crozier)) = (
fern_docs_without_lifted(pair.fern, names),
crozier_docs_without_lifted(pair.crozier, names),
) else {
return Ok(None);
};
if !fern
.iter()
.zip(&crozier)
.all(|(left, right)| *left == right)
|| fern.len() != crozier.len()
{
let methods: BTreeSet<String> = lifted_methods(pair.context, names)
.into_iter()
.filter(|method| {
crozier
.iter()
.any(|line| line.contains(&format!(".{method}()")))
})
.collect();
if without_lifted_method_placeholders(&fern, &methods) != crozier {
return Ok(None);
}
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn date_header_constructor_example(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !pair.rel.ends_with("client.py") || pair.rel.ends_with("raw_client.py") {
return Ok(None);
}
let names: BTreeSet<&str> = pair
.fern
.iter()
.filter_map(|line| {
line.trim()
.split_once(": typing.Optional[dt.date] = None")
.map(|(name, _)| name)
})
.collect();
if names.is_empty() {
return Ok(None);
}
let mut changed = false;
let fern: Vec<String> = pair
.fern
.iter()
.map(|line| {
let mut line = (*line).to_string();
for name in &names {
let placeholder = format!("{name}=\"YOUR_{}\"", name.to_uppercase());
if line.contains(&placeholder) {
changed = true;
line = line.replace(
&placeholder,
&format!("{name}=datetime.date.fromisoformat(\"2023-01-15\")"),
);
}
}
line
})
.collect();
let without_example_import = |lines: &[String]| {
let mut result = Vec::new();
let mut index = 0;
while index < lines.len() {
if lines[index].trim() == "import datetime"
&& index + 2 < lines.len()
&& lines[index + 1].is_empty()
&& lines[index + 2].trim_start().starts_with("from ")
{
index += if index > 0 && lines[index - 1].trim() == "import asyncio" {
1
} else {
2
};
} else {
result.push(lines[index].clone());
index += 1;
}
}
result
};
let crozier: Vec<String> = pair
.crozier
.iter()
.map(|line| (*line).to_string())
.collect();
if !changed || without_example_import(&fern) != without_example_import(&crozier) {
return Ok(None);
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn lifted_methods(context: &Context, names: &BTreeSet<String>) -> BTreeSet<String> {
let Some((_, root)) = &context.roots else {
return BTreeSet::new();
};
let mut methods = BTreeSet::new();
for (_, source) in python_sources(root) {
let mut method = None;
for line in source.lines() {
let trimmed = line.trim_start();
if let Some(rest) = trimmed
.strip_prefix("def ")
.or_else(|| trimmed.strip_prefix("async def "))
{
method = rest.split_once('(').map(|(name, _)| name.to_string());
}
if names
.iter()
.any(|name| line.contains(&format!("self._client_wrapper._{name}")))
{
if let Some(method) = &method {
methods.insert(method.clone());
}
}
}
}
methods
}
fn without_lifted_method_placeholders(lines: &[&str], methods: &BTreeSet<String>) -> Vec<String> {
let mut kept: Vec<String> = Vec::new();
for line in lines {
let mut line = (*line).to_string();
for method in methods {
line = line.replace(&format!(".{method}(..., "), &format!(".{method}("));
line = line.replace(&format!(".{method}(...)"), &format!(".{method}()"));
line = line.replace(&format!(">{method}</a>(...)"), &format!(">{method}</a>()"));
}
if line.trim() == ")"
&& kept.last().is_some_and(|previous| {
methods
.iter()
.any(|method| previous.trim_end().ends_with(&format!(".{method}(")))
})
{
if let Some(previous) = kept.last_mut() {
previous.push(')');
}
} else {
kept.push(line);
}
}
kept
}
fn sdk_variable_docs_examples(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !DOCS_FILES.contains(&pair.rel) {
return Ok(None);
}
let names = pair.context.crozier_lifted_parameters();
let methods = lifted_methods(pair.context, names);
let Some(crozier) = crozier_docs_without_lifted(pair.crozier, names) else {
return Ok(None);
};
let mut fern = Vec::new();
let mut index = 0;
while index < pair.fern.len() {
if let Some(length) = lifted_parameter_block(pair.fern, index, names) {
index += length;
continue;
}
let mut line = pair.fern[index].to_string();
for method in &methods {
line = line.replace(&format!(".{method}(..., "), &format!(".{method}("));
line = line.replace(&format!(".{method}(...)"), &format!(".{method}()"));
line = line.replace(&format!(">{method}</a>(...)"), &format!(">{method}</a>()"));
}
fern.push(line);
index += 1;
}
if methods.is_empty() || fern != crozier {
return Ok(None);
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn header_doc_names(wire: &str) -> [String; 2] {
[
crate::naming::field_name(crate::ir::header_param_stem(wire)),
crate::naming::field_name(wire)
.trim_end_matches('_')
.to_string(),
]
}
fn constant_header_docs_arguments(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if pair.rel != "reference.md" {
return Ok(None);
}
let constants = pair.context.crozier_constant_headers();
if constants.is_empty() {
return Ok(None);
}
let documented: BTreeSet<String> = constants
.iter()
.flat_map(|(wire, _)| header_doc_names(wire))
.collect();
let lines = pair.fern;
let mut kept: Vec<&str> = Vec::with_capacity(lines.len());
let mut removed = false;
let mut index = 0;
while index < lines.len() {
let block = parameter_block(lines, index, |line| {
line.strip_prefix("**")
.and_then(|rest| rest.split_once(":** `typing.Literal`"))
.is_some_and(|(name, after)| {
documented.contains(name) && (after == " " || after.starts_with(" — "))
})
});
if let Some(length) = block {
index += length;
removed = true;
continue;
}
kept.push(lines[index]);
index += 1;
}
if !removed || kept != pair.crozier {
return Ok(None);
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn nullable_items_docs(pair: &Pair<'_>) -> Result<Option<Region>, String> {
if !DOCS_FILES.contains(&pair.rel) {
return Ok(None);
}
let names = pair.context.reference_nullable_items();
if names.is_empty() {
return Ok(None);
}
let mut fern_form: Vec<String> = Vec::with_capacity(pair.crozier.len());
let mut list: Option<String> = None;
for line in pair.crozier {
if let Some(close) = &list {
if *line == close {
list = None;
fern_form.push((*line).to_string());
} else {
let indent = &line[..line.len() - line.trim_start().len()];
fern_form.push(format!("{indent}None"));
}
continue;
}
let trimmed = line.trim_start();
let indent = &line[..line.len() - trimmed.len()];
if let Some(name) = trimmed.strip_suffix("=[") {
if names.contains(name) {
list = Some(format!("{indent}],"));
}
fern_form.push((*line).to_string());
continue;
}
let restored = documented_parameter(line).and_then(|(name, annotation)| {
if !names.contains(name) {
return None;
}
let item = annotation
.strip_prefix("typing.Optional[typing.Union[")?
.split_once(", typing.Sequence[")?
.0;
let (fern, crozier) = nullable_items_types(item);
(crozier == annotation).then(|| line.replacen(&crozier, &fern, 1))
});
fern_form.push(restored.unwrap_or_else(|| (*line).to_string()));
}
if fern_form.len() != pair.fern.len()
|| fern_form
.iter()
.zip(pair.fern)
.any(|(left, right)| left != right)
|| fern_form
.iter()
.zip(pair.crozier)
.all(|(left, right)| left == right)
{
return Ok(None);
}
Ok(differing_window(pair.fern, pair.crozier))
}
fn is_client_wrapper(rel: &str) -> bool {
rel == "core/client_wrapper.py" || rel.ends_with("/core/client_wrapper.py")
}
fn sdk_identity_header_prefix(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
is_client_wrapper(pair.rel)
&& fern.trim_start().starts_with("\"X-Fern-")
&& fern.replacen("\"X-Fern-", "\"X-Crozier-", 1) == crozier
}
fn identity_pair_line<'l>(line: &'l str, prefix: &str) -> Option<(&'l str, &'l str, &'l str)> {
let indent = &line[..line.len() - line.trim_start().len()];
let rest = line.trim_start().strip_prefix('"')?.strip_prefix(prefix)?;
let (header, value) = ["SDK-Name\": ", "SDK-Version\": "]
.iter()
.find_map(|header| rest.strip_prefix(header).map(|value| (*header, value)))?;
let value = value.strip_suffix(',')?;
(value.len() >= 2 && value.starts_with('"') && value.ends_with('"'))
.then_some((indent, header, value))
}
fn is_crozier_identity_value(pair: &Pair<'_>, header: &str, value: &str) -> bool {
let expected = if header.starts_with("SDK-Name") {
match pair.context.crozier_project() {
Some(project) => format!("\"{project}\""),
None => return false,
}
} else {
format!("\"{}\"", crate::emit::DEFAULT_SDK_VERSION)
};
value == expected
}
fn sdk_name_version_header(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if !is_client_wrapper(pair.rel) {
return false;
}
match (
identity_pair_line(fern, "X-Fern-"),
identity_pair_line(crozier, "X-Crozier-"),
) {
(Some(fern), Some(crozier)) => {
fern.0 == crozier.0
&& fern.1 == crozier.1
&& fern.2 != crozier.2
&& is_crozier_identity_value(pair, crozier.1, crozier.2)
}
_ => false,
}
}
fn sdk_name_version_header_added(pair: &Pair<'_>, crozier: &str) -> bool {
is_client_wrapper(pair.rel)
&& identity_pair_line(crozier, "X-Crozier-")
.is_some_and(|(_, header, value)| is_crozier_identity_value(pair, header, value))
&& !pair
.fern
.iter()
.any(|line| identity_pair_line(line, "X-Fern-").is_some())
}
fn example_file(rel: &str) -> bool {
DOCS_FILES.contains(&rel) || high_level_client(rel)
}
fn keyword_argument_line(line: &str) -> Option<(&str, &str, &str)> {
let trimmed = line.trim_start();
let indent = &line[..line.len() - trimmed.len()];
let (name, value) = trimmed.strip_suffix(',')?.split_once('=')?;
python_identifier(name).then_some((indent, name, value))
}
fn source_operations(
source: &crate::openapi::OpenApi,
) -> impl Iterator<Item = (&crate::openapi::Operation, Vec<&crate::openapi::Parameter>)> {
source.paths.values().flat_map(|item| {
item.operations().into_iter().map(move |(_, operation)| {
let parameters = operation
.parameters
.iter()
.chain(item.parameters.iter().filter(|shared| {
!operation
.parameters
.iter()
.any(|own| own.name == shared.name && own.location == shared.location)
}))
.collect();
(operation, parameters)
})
})
}
fn argument_names(wire: &str, keyword: &str) -> bool {
[
crate::naming::field_name(wire),
crate::naming::request_field_name(wire),
crate::naming::model_field_name(wire),
]
.iter()
.any(|name| name.trim_end_matches('_') == keyword.trim_end_matches('_'))
}
fn declared_schema_example(schema: &crate::openapi::Schema) -> Option<&serde_json::Value> {
schema.example.as_ref().or_else(|| schema.examples.first())
}
fn declared_parameter_examples(
parameter: &crate::openapi::Parameter,
) -> impl Iterator<Item = &serde_json::Value> {
parameter
.example
.iter()
.chain(
parameter
.examples
.values()
.filter_map(|example| example.value.as_ref()),
)
.chain(parameter.schema.as_ref().and_then(declared_schema_example))
}
fn is_query(parameter: &crate::openapi::Parameter) -> bool {
parameter.location == Some(crate::openapi::ParameterLocation::Query)
}
fn is_path(parameter: &crate::openapi::Parameter) -> bool {
parameter.location == Some(crate::openapi::ParameterLocation::Path)
}
fn anyof_path_param_schema_example(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if !example_file(pair.rel) {
return false;
}
let (Some((fern_indent, fern_name, fern_value)), Some((indent, name, value))) =
(keyword_argument_line(fern), keyword_argument_line(crozier))
else {
return false;
};
if fern_indent != indent || fern_name != name || fern_value == value {
return false;
}
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
source_operations(source).any(|(_, parameters)| {
parameters.iter().any(|parameter| {
let Some(schema) = parameter.schema.as_ref() else {
return false;
};
is_path(parameter)
&& argument_names(¶meter.name, name)
&& fern_value == format!("{:?}", parameter.name)
&& schema.reference.is_none()
&& (schema.any_of.is_some() || schema.one_of.is_some())
&& declared_schema_example(schema)
.filter(|example| example.is_string())
.and_then(crate::ir::example_literal)
.is_some_and(|literal| literal == value)
})
})
}
fn text_only_success(operation: &crate::openapi::Operation) -> bool {
let mut media = operation
.responses
.iter()
.filter(|(status, _)| status.starts_with('2'))
.flat_map(|(_, response)| response.content.keys())
.peekable();
media.peek().is_some() && media.all(|media_type| media_type.starts_with("text/"))
}
fn text_response_optional_query_examples(pair: &Pair<'_>, crozier: &str) -> bool {
if !example_file(pair.rel) {
return false;
}
let Some((_, name, value)) = keyword_argument_line(crozier) else {
return false;
};
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
source_operations(source)
.filter(|(operation, _)| text_only_success(operation))
.any(|(_, parameters)| {
parameters.iter().any(|parameter| {
is_query(parameter)
&& parameter.required != Some(true)
&& argument_names(¶meter.name, name)
&& declared_parameter_examples(parameter)
.filter_map(crate::ir::example_literal)
.any(|literal| literal == value)
})
})
}
fn list_argument_at<'l>(lines: &[&'l str], at: usize) -> Option<(&'l str, Vec<&'l str>)> {
let line = *lines.get(at)?;
if let Some((_, name, value)) = keyword_argument_line(line) {
let items = value.strip_prefix('[')?.strip_suffix(']')?;
let items = if items.is_empty() {
Vec::new()
} else {
items.split(", ").collect()
};
return Some((name, items));
}
let depth = |line: &str| line.len() - line.trim_start().len();
let opener = (0..=at)
.rev()
.find(|&index| lines[index].ends_with("=[") && depth(lines[index]) <= depth(line))?;
let indent = &lines[opener][..depth(lines[opener])];
let name = lines[opener].trim_start().strip_suffix("=[")?;
let closer = (opener + 1..lines.len()).find(|&index| lines[index] == format!("{indent}],"))?;
if at > closer || !python_identifier(name) {
return None;
}
let items = lines[opener + 1..closer]
.iter()
.map(|item| {
let item = item.strip_prefix(indent)?.strip_prefix(" ")?;
(!item.starts_with(' ') && !item.is_empty()).then(|| item.trim_end_matches(','))
})
.collect::<Option<Vec<_>>>()?;
Some((name, items))
}
fn item_is_value(item: &str, expected: &serde_json::Value) -> bool {
match expected {
serde_json::Value::Number(number) => item
.parse::<f64>()
.ok()
.zip(number.as_f64())
.is_some_and(|(left, right)| left == right),
serde_json::Value::String(_) => {
crate::ir::example_literal(expected).as_deref() == Some(item)
}
serde_json::Value::Bool(value) => item == if *value { "True" } else { "False" },
_ => false,
}
}
fn optional_lowered_required_query_array_example(pair: &Pair<'_>, at: usize) -> bool {
if !example_file(pair.rel) {
return false;
}
let Some((name, items)) = list_argument_at(pair.crozier, at) else {
return false;
};
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
source_operations(source).any(|(_, parameters)| {
parameters.iter().any(|parameter| {
is_query(parameter)
&& parameter.required == Some(true)
&& parameter
.schema
.as_ref()
.and_then(|schema| schema.ty.as_ref()?.primary())
== Some("array")
&& argument_names(¶meter.name, name)
&& declared_parameter_examples(parameter).any(|example| {
example.as_array().is_some_and(|values| {
!values.is_empty()
&& values.len() == items.len()
&& values
.iter()
.zip(&items)
.all(|(value, item)| item_is_value(item, value))
})
})
})
})
}
fn collapsed_property_descriptions(source: &crate::openapi::OpenApi) -> Vec<(String, String)> {
let mut found = Vec::new();
let mut visit = |schema: &crate::openapi::Schema| {
for (name, property) in &schema.properties {
let Some(members) = property.any_of.as_ref().or(property.one_of.as_ref()) else {
continue;
};
let alternatives: Vec<_> = members
.iter()
.filter(|member| member.ty.as_ref().and_then(|ty| ty.primary()) != Some("null"))
.collect();
let Some(description) = property.description.as_deref() else {
continue;
};
if property.reference.is_none()
&& alternatives.len() >= 2
&& alternatives.iter().all(|member| {
member.reference.is_none()
&& member.properties.is_empty()
&& member.items.is_none()
})
&& alternatives[0].description.is_none()
{
found.push((
name.clone(),
description.trim_end_matches(['\r', '\n']).to_string(),
));
}
}
};
visit_source_schemas(Some(source), &mut visit);
found
}
fn collapsed_anyof_property_description(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
let properties = collapsed_property_descriptions(source);
if pair.rel == "reference.md" {
let Some(head) = fern.strip_suffix(' ') else {
return false;
};
let Some(name) = head
.strip_prefix("**")
.and_then(|rest| rest.split_once(":** `"))
.map(|(name, _)| name)
else {
return false;
};
return properties.iter().any(|(wire, description)| {
argument_names(wire, name)
&& crozier
== format!(
"{head}{}",
crate::emit::reference_param_suffix(Some(description))
)
});
}
pair.rel.ends_with(".py")
&& crozier.strip_suffix(" = pydantic.Field()") == Some(fern)
&& fern.trim_start().split_once(": ").is_some_and(|(name, _)| {
fern.starts_with(" ")
&& properties
.iter()
.any(|(wire, _)| argument_names(wire, name))
})
}
fn collapsed_anyof_property_description_line(pair: &Pair<'_>, at: usize) -> bool {
if !pair.rel.ends_with(".py") {
return false;
}
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
let lines = pair.crozier;
let described = |name: &str, text: &str| {
collapsed_property_descriptions(source)
.iter()
.any(|(wire, description)| argument_names(wire, name) && description.trim() == text)
};
if let Some(field) = (at.saturating_sub(64)..=at).rev().find(|&index| {
lines[index].starts_with(" ")
&& !lines[index].starts_with(" ")
&& lines[index].ends_with(" = pydantic.Field()")
}) {
let open = field + 1;
if lines.get(open) == Some(&" \"\"\"") {
if let Some(close) = (open + 1..lines.len()).find(|&index| lines[index] == " \"\"\"")
{
let blank = close + 1;
let text = lines[open + 1..close]
.iter()
.map(|line| line.strip_prefix(" ").unwrap_or(line))
.collect::<Vec<_>>()
.join("\n");
let name = lines[field]
.trim_start()
.split_once(": ")
.map_or("", |(name, _)| name);
if (open..=blank).contains(&at)
&& lines.get(blank).is_some_and(|line| line.is_empty())
&& described(name, text.trim())
{
return true;
}
}
}
}
let indent = lines[at].len() - lines[at].trim_start().len();
if lines[at].trim().is_empty() || indent < 4 {
return false;
}
let Some(header) = (0..at).rev().find(|&index| {
let line = lines[index];
line.trim().is_empty() || line.len() - line.trim_start().len() < indent
}) else {
return false;
};
let header_line = lines[header];
if header_line.len() - header_line.trim_start().len() + 4 != indent {
return false;
}
let Some((name, _)) = header_line.trim_start().split_once(" : ") else {
return false;
};
let end = (header + 1..lines.len())
.find(|&index| {
lines[index].trim().is_empty()
|| lines[index].len() - lines[index].trim_start().len() < indent
})
.unwrap_or(lines.len());
let text = lines[header + 1..end]
.iter()
.map(|line| &line[indent..])
.collect::<Vec<_>>()
.join("\n");
described(name, text.trim())
}
fn crozier_modules(context: &Context) -> BTreeSet<String> {
context
.crozier_paths()
.iter()
.filter_map(|path| {
let rest = path.strip_prefix("src/")?;
let (_, module) = rest.split_once('/')?;
module.strip_suffix("/client.py").map(str::to_string)
})
.collect()
}
fn emoji_tag_reference_identifiers(pair: &Pair<'_>, fern: &str, crozier: &str) -> bool {
if !DOCS_FILES.contains(&pair.rel) || fern == crozier {
return false;
}
let Some(source) = pair.context.source_document.as_ref() else {
return false;
};
let modules = crozier_modules(pair.context);
source
.paths
.values()
.flat_map(|item| item.operations())
.flat_map(|(_, operation)| &operation.tags)
.any(|tag| {
let raw = tag
.to_lowercase()
.split_whitespace()
.collect::<Vec<_>>()
.join("_");
let identifier: String = raw
.chars()
.filter(|character| character.is_ascii_alphanumeric() || *character == '_')
.collect::<String>()
.trim_matches('_')
.to_string();
raw != identifier
&& python_identifier(&identifier)
&& modules.contains(&identifier)
&& crozier.contains(identifier.as_str())
&& crozier.replace(identifier.as_str(), &raw) == fern
})
}
#[cfg(test)]
mod tests {
use super::*;
fn pair<'a>(rel: &'a str, fern: &'a [&'a str], crozier: &'a [&'a str]) -> Pair<'a> {
static EMPTY: OnceLock<Context> = OnceLock::new();
Pair {
rel,
fern,
crozier,
context: EMPTY.get_or_init(Context::default),
}
}
#[test]
fn the_compiled_catalog_is_valid_and_every_entry_has_a_rule() {
let entries = parse(CATALOG_SOURCE).unwrap_or_else(|failures| panic!("{failures:#?}"));
assert_eq!(
entries
.iter()
.map(|entry| entry.id.as_str())
.collect::<Vec<_>>(),
RULE_IDS
);
assert_eq!(catalog(), entries.as_slice());
assert!(find("readme-client-class-casing").is_some());
assert!(find("no-such-departure").is_none());
assert!(entries.iter().any(|entry| entry.kind == Kind::FernDefect));
}
#[test]
fn every_evidence_note_is_committed() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
for entry in catalog() {
assert!(
root.join(&entry.evidence).is_file(),
"departure `{}`: evidence {} is not committed",
entry.id,
entry.evidence
);
}
}
fn entry_text(id: &str, kind: &str, evidence: &str) -> String {
let keys = " trigger: t\n fern: f\n crozier: c\n reason: r\n";
format!("- id: {id}\n kind: {kind}\n{keys} evidence: {evidence}\n")
}
fn failures_of(text: &str) -> Vec<String> {
parse(text).expect_err("the catalog is refused")
}
fn assert_refused(text: &str, says: &str) {
let failures = failures_of(text);
assert!(
failures.iter().any(|failure| failure.contains(says)),
"expected a failure saying {says:?}: {failures:#?}"
);
}
#[test]
fn a_broken_catalog_is_refused_naming_the_entry() {
let evidence = "docs/departures/evidence/x.md";
let good = |id: &str| entry_text(id, "branding", evidence);
assert_refused(
&format!(
"{}{}",
good("sdk-name-version-headers"),
good("sdk-identity-header-prefix")
),
"departure `sdk-identity-header-prefix`: follows `sdk-name-version-headers`",
);
assert_refused(
&format!(
"{}{}",
good("sdk-identity-header-prefix"),
good("sdk-identity-header-prefix")
),
"the id appears twice",
);
assert_refused(&good("Not_A_Slug"), "departure `Not_A_Slug`: the id is not");
assert_refused(&good("unruled-departure"), "no rule of that id");
assert_refused(
&good("sdk-identity-header-prefix"),
"departure rule `readme-client-class-casing` has no catalog entry",
);
assert_refused(
&good("sdk-identity-header-prefix").replace("reason: r", "reason: ''"),
"`reason` is empty",
);
for evidence in ["docs/elsewhere/x.md", "docs/departures/evidence/../x.md"] {
assert_refused(
&entry_text("sdk-identity-header-prefix", "branding", evidence),
"is not a note under docs/departures/evidence/",
);
}
assert_refused(
&entry_text("sdk-identity-header-prefix", "cosmetic", evidence),
"holding exactly the keys id, kind, trigger, fern, crozier, reason, evidence",
);
assert_refused(
&format!("{} extra: x\n", good("sdk-identity-header-prefix")),
"unknown field `extra`",
);
assert_refused(
&good("sdk-identity-header-prefix").replace(" fern: f\n", ""),
"missing field `fern`",
);
}
#[test]
fn the_reference_renders_every_kind_and_entry() {
let rendered = render_reference(catalog());
assert!(rendered.starts_with(REFERENCE_MARKERS.0), "{rendered}");
assert!(
rendered.trim_end().ends_with(REFERENCE_MARKERS.1),
"{rendered}"
);
for kind in Kind::ALL {
assert!(rendered.contains(&format!("| `{}` |", kind.as_str())));
}
for entry in catalog() {
assert!(rendered.contains(&format!("### `{}`", entry.id)));
assert!(rendered.contains(&format!("(../../{})", entry.evidence)));
}
}
#[test]
fn the_rendered_reference_is_current() {
let path =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("docs/departures/README.md");
let page = std::fs::read_to_string(&path).expect("docs/departures/README.md");
let (begin, end) = REFERENCE_MARKERS;
let (Some(start), Some(stop)) = (page.find(begin), page.find(end)) else {
panic!("docs/departures/README.md holds no generated catalog markers");
};
let committed = &page[start..stop + end.len() + 1];
let rendered = render_reference(catalog());
if std::env::var_os("CROZIER_UPDATE_DEPARTURES").is_some() {
std::fs::write(&path, page.replacen(committed, &rendered, 1)).unwrap();
return;
}
assert!(
committed == rendered,
concat!(
"docs/departures/README.md's catalog differs from assets/departures.yml; ",
"regenerate it with `CROZIER_UPDATE_DEPARTURES=1 cargo test --lib departures`"
)
);
}
#[test]
fn classes_are_read_from_top_level_python_class_lines_only() {
let context = Context::from_sources(
[
(
"client.py",
"class Acme:\n class Inner:\nclass Async_2(x):\n",
),
("README.md", "class Ignored:\n"),
],
[("a.py", "class Spaced:\nclass :\n")],
);
assert_eq!(
context.reference_classes(),
&BTreeSet::from(["Acme".to_string(), "Async_2".to_string()])
);
assert!(context.crozier_classes().is_empty());
let reference = tempfile::tempdir().unwrap();
let crozier = tempfile::tempdir().unwrap();
std::fs::create_dir_all(reference.path().join("pkg")).unwrap();
std::fs::write(reference.path().join("pkg/client.py"), "class Acme:\n").unwrap();
std::fs::write(reference.path().join("README.md"), "class Not:\n").unwrap();
let trees = Context::from_trees(reference.path(), crozier.path());
assert_eq!(
trees.reference_classes(),
&BTreeSet::from(["Acme".to_string()])
);
assert!(trees.crozier_classes().is_empty());
assert!(Context::default().reference_classes().is_empty());
}
type Tree = Vec<(&'static str, String)>;
fn empty_namespace_trees() -> (Tree, Tree) {
let client = |example: &str| {
format!(
"from ..core.client_wrapper import SyncClientWrapper\nfrom .raw_client import RawClient\n\n\nclass Client:\n def ping(self) -> None:\n \"\"\"\n Examples\n --------\n {example}\n \"\"\"\n return None\n"
)
};
let raw =
"from ..core.client_wrapper import SyncClientWrapper\n\n\nclass RawClient:\n pass\n";
let hoisted = "from ...core.pydantic_utilities import UniversalBaseModel\n\n\nclass Ping(UniversalBaseModel):\n pass\n";
let reference = vec![
("src/fern/client.py", client("client..ping()")),
("src/fern/raw_client.py", raw.to_string()),
(
"src/fern/__init__.py",
"\n\nfrom .types import Ping\n".to_string(),
),
("src/fern/types/ping.py", hoisted.to_string()),
(
"src/fern/types/__init__.py",
"from .ping import Ping\n".to_string(),
),
];
let crozier = vec![
(
"src/fern/client.py",
"class FernApi:\n @property\n def _(self):\n pass\n".to_string(),
),
(
"src/fern/__init__.py",
"from .client import FernApi\n".to_string(),
),
(
"src/fern/types/__init__.py",
"from .ping import Ping\n".to_string(),
),
("src/fern/_/client.py", client("client._.ping()")),
("src/fern/_/raw_client.py", raw.to_string()),
(
"src/fern/_/__init__.py",
"from .types import Ping\n".to_string(),
),
("src/fern/_/types/ping.py", hoisted.to_string()),
(
"src/fern/_/types/__init__.py",
"from .ping import Ping\n".to_string(),
),
];
(reference, crozier)
}
fn write_tree(files: &[(&str, String)]) -> tempfile::TempDir {
let root = tempfile::tempdir().unwrap();
for (rel, text) in files {
let path = root.path().join(rel);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, text).unwrap();
}
root
}
#[test]
fn the_empty_namespace_rule_holds_only_where_crozier_moves_fern_s_files_unchanged() {
let (reference, crozier) = empty_namespace_trees();
let (fern_tree, crozier_tree) = (write_tree(&reference), write_tree(&crozier));
let context = Context::from_trees(fern_tree.path(), crozier_tree.path());
let found = context.empty_namespace();
assert_eq!(
found.crozier_only,
BTreeSet::from(
[
"src/fern/_/__init__.py",
"src/fern/_/client.py",
"src/fern/_/raw_client.py",
"src/fern/_/types/__init__.py",
"src/fern/_/types/ping.py"
]
.map(String::from)
)
);
assert_eq!(
found.reference_only,
BTreeSet::from(["src/fern/raw_client.py", "src/fern/types/ping.py"].map(String::from))
);
assert!(empty_namespace_package_file(
&context,
"src/fern/raw_client.py",
Side::Reference
));
assert!(!empty_namespace_package_file(
&context,
"src/fern/raw_client.py",
Side::Crozier
));
let (fern, ours) = (["class Client:", ""], ["class FernApi:", "", ""]);
let root = Pair {
rel: "src/fern/client.py",
fern: &fern,
crozier: &ours,
context: &context,
};
assert_eq!(
empty_namespace_package_root(&root).unwrap(),
Some(Region {
fern: 0..2,
crozier: 0..3
})
);
let elsewhere = Pair {
rel: "src/fern/types/__init__.py",
..root
};
assert_eq!(empty_namespace_package_root(&elsewhere).unwrap(), None);
let unfound = |edit: &dyn Fn(&mut Tree, &mut Tree)| {
let (mut reference, mut crozier) = empty_namespace_trees();
edit(&mut reference, &mut crozier);
let (fern_tree, crozier_tree) = (write_tree(&reference), write_tree(&crozier));
let context = Context::from_trees(fern_tree.path(), crozier_tree.path());
let found = context.empty_namespace();
found.crozier_only.is_empty()
&& found.reference_only.is_empty()
&& found.replaced.is_empty()
};
assert!(unfound(&|reference, _| {
reference[0].1 = reference[0].1.replace("from ..core.", "from .core.");
}));
assert!(unfound(&|_, crozier| crozier[4]
.1
.push_str("# kept\nx = 1\n")));
assert!(unfound(
&|_, crozier| crozier[3].1 = crozier[3].1.replace("return None", "return 1")
));
assert!(Context::default().empty_namespace().replaced.is_empty());
}
#[test]
fn examples_are_the_only_lines_without_examples_drops() {
let text = "def f():\n \"\"\"\n Doc.\n\n Examples\n --------\n client..f()\n \"\"\"\n return 1\n";
assert_eq!(
without_examples(text),
[
"def f():",
" \"\"\"",
" Doc.",
"",
" \"\"\"",
" return 1"
]
);
assert_eq!(without_examples("Examples\nx"), ["Examples", "x"]);
}
#[test]
fn the_stream_heading_rule_holds_only_on_the_declared_iterator() {
let client = "class FernApi:\n def watch(\n self, *, request_options: typing.Optional[RequestOptions] = None\n ) -> typing.Iterator[Tick]:\n pass\n def at(self) -> typing.Iterator[dt.datetime]:\n pass\n def raw(self) -> typing.Iterator[bytes]:\n pass\nclass AsyncFernApi:\n async def watch(self) -> typing.AsyncIterator[Tick]:\n pass\n";
let context = Context::from_sources(
[("src/fern/client.py", client)],
[("src/fern/client.py", client)],
);
let lines: [&str; 0] = [];
let pair = Pair {
rel: "reference.md",
fern: &lines,
crozier: &lines,
context: &context,
};
let heading = |method: &str, annotation: &str| {
format!("<details><summary><code>client.<a href=\"src/fern/client.py\">{method}</a>(...) -> {annotation}</code></summary>")
};
let holds = |fern: &str, crozier: &str| stream_reference_return_type(&pair, fern, crozier);
let fern = heading("watch", "typing.Iterator[bytes]");
assert!(holds(&fern, &heading("watch", "typing.Iterator[Tick]")));
assert!(holds(
&heading("at", "typing.Iterator[bytes]"),
&heading("at", "typing.Iterator[datetime.datetime]")
));
assert!(!holds(&fern, &heading("watch", "typing.Iterator[Other]")));
assert!(!holds(&fern, &heading("watch", "Tick")));
assert!(!holds(
&heading("raw", "typing.Iterator[bytes]"),
&heading("raw", "typing.Iterator[bytes]")
));
assert!(!holds(
&heading("missing", "typing.Iterator[bytes]"),
&heading("missing", "typing.Iterator[Tick]")
));
assert!(!holds(&fern, &heading("other", "typing.Iterator[Tick]")));
assert!(!holds(
&heading("watch", "Tick"),
&heading("watch", "typing.Iterator[Tick]")
));
let readme = Pair {
rel: "README.md",
..pair
};
assert!(!stream_reference_return_type(
&readme,
&fern,
&heading("watch", "typing.Iterator[Tick]")
));
let flat = Context::from_sources([("client.py", client)], [("client.py", client)]);
let flat = Pair {
context: &flat,
..pair
};
assert!(stream_reference_return_type(
&flat,
&fern,
&heading("watch", "typing.Iterator[Tick]")
));
assert!(!stream_reference_return_type(
&flat,
&fern,
&heading("watch", "typing.Iterator[Other]")
));
let alone = Context::default();
let alone = Pair {
context: &alone,
..pair
};
assert!(!stream_reference_return_type(
&alone,
&fern,
&heading("watch", "typing.Iterator[Tick]")
));
}
#[test]
fn the_casing_rule_holds_only_where_crozier_names_the_defined_class() {
let context = Context::from_sources(
[("client.py", "class LanternHarborApi:\n")],
[("client.py", "class LanternHarborApi:\n")],
);
let lines: [&str; 0] = [];
let pair = Pair {
rel: "README.md",
fern: &lines,
crozier: &lines,
context: &context,
};
let holds = |fern: &str, crozier: &str| readme_client_class_casing(&pair, fern, crozier);
assert!(holds(
"from LanternHarbor import LanternharborApi",
"from LanternHarbor import LanternHarborApi"
));
assert!(!holds(
"from LanternHarbor import LanternharborApi",
"from LanternHarbor import LanternHarborApi, X"
));
assert!(!holds(
"client = LanternharborApi()",
"client = LanternHarborApi( )"
));
assert!(!holds("x = Lanternharbor", "x = LanternHarbor"));
assert!(!holds("x = LanternHarborApi", "x = LanternHarborApi"));
assert!(!holds("x = LanternHarborApi", "x = lanternharborapi"));
let defined = Context::from_sources(
[(
"client.py",
"class LanternharborApi:\nclass LanternHarborApi:\n",
)],
[("client.py", "class LanternHarborApi:\n")],
);
let pair = Pair {
context: &defined,
..pair
};
assert!(!readme_client_class_casing(
&pair,
"x = LanternharborApi",
"x = LanternHarborApi"
));
let other = Pair {
rel: "docs/README.md",
context: &context,
..pair
};
assert!(!readme_client_class_casing(
&other,
"x = LanternharborApi",
"x = LanternHarborApi"
));
}
fn lifted_context() -> Context {
Context::from_sources(
[],
[(
"src/acme/layers/raw_client.py",
"f\"{encode_path_param(self._client_wrapper._edition)}/layers\",\n",
)],
)
}
#[test]
fn lifted_parameters_are_read_from_routes_reading_the_client_wrapper() {
assert_eq!(
lifted_context().crozier_lifted_parameters(),
&BTreeSet::from(["edition".to_string()])
);
let other = Context::from_sources(
[("a.py", "encode_path_param(self._client_wrapper._edition)\n")],
[
("README.md", "encode_path_param(self._client_wrapper._a)\n"),
(
"b.py",
"self._client_wrapper._b\nencode_path_param(self._client_wrapper._c.x)\n",
),
],
);
assert!(other.crozier_lifted_parameters().is_empty());
let crozier = tempfile::tempdir().unwrap();
std::fs::write(
crozier.path().join("raw_client.py"),
"encode_path_param(self._client_wrapper._tenant)\n",
)
.unwrap();
let reference = tempfile::tempdir().unwrap();
assert_eq!(
Context::from_trees(reference.path(), crozier.path()).crozier_lifted_parameters(),
&BTreeSet::from(["tenant".to_string()])
);
assert!(Context::default().crozier_lifted_parameters().is_empty());
}
const LIFTED_FERN: &str = "client = FernApi()\n\nclient.layers.create_layer(\n edition=\"v2\",\n title=\"title\",\n)\n\n<dl>\n<dd>\n\n**edition:** `str` \n \n</dd>\n</dl>\n\n<dl>\n<dd>\n\n**request:** `Layer` \n";
const LIFTED_CROZIER: &str = "client = FernApi(\n edition=\"v2\",\n)\n\nclient.layers.create_layer(\n title=\"title\",\n)\n\n<dl>\n<dd>\n\n**request:** `Layer` \n";
fn docs_region(rel: &str, fern: &str, crozier: &str, context: &Context) -> Option<Region> {
let fern: Vec<&str> = fern.split('\n').collect();
let crozier: Vec<&str> = crozier.split('\n').collect();
let pair = Pair {
rel,
fern: &fern,
crozier: &crozier,
context,
};
lifted_base_path_docs_examples(&pair).unwrap()
}
#[test]
fn the_lifted_docs_rule_takes_exactly_the_three_constructs() {
let context = lifted_context();
assert_eq!(
docs_region("reference.md", LIFTED_FERN, LIFTED_CROZIER, &context),
Some(Region {
fern: 0..14,
crozier: 0..7,
})
);
assert!(docs_region(
"README.md",
"client = AsyncFernApi(\n environment=E,\n)\n await client.layers.get(\n edition=\"edition\",\n )",
"client = AsyncFernApi(\n edition=\"YOUR_EDITION\",\n environment=E,\n)\n await client.layers.get(\n )",
&context,
)
.is_some());
for (fern, crozier) in [
(format!("{LIFTED_FERN}x"), format!("{LIFTED_CROZIER}y")),
(
LIFTED_FERN.replace(" title=", " opacity=1,\n title="),
LIFTED_CROZIER.to_string(),
),
(
LIFTED_FERN.to_string(),
LIFTED_CROZIER.replace(" title=", " edition=\"v2\",\n title="),
),
(
LIFTED_FERN.replace("**edition:**", "**tenant:**"),
LIFTED_CROZIER.to_string(),
),
(
LIFTED_FERN.to_string(),
LIFTED_CROZIER.replace(" edition=\"v2\",\n", " tenant=\"v2\",\n"),
),
(
LIFTED_FERN.replace("client = FernApi()", "client = FernApi(1)"),
LIFTED_CROZIER.to_string(),
),
] {
assert_eq!(
docs_region("reference.md", &fern, &crozier, &context),
None,
"{fern}\n---\n{crozier}"
);
}
assert_eq!(
docs_region("client.py", LIFTED_FERN, LIFTED_CROZIER, &context),
None
);
assert_eq!(
docs_region(
"README.md",
LIFTED_FERN,
LIFTED_CROZIER,
&Context::default()
),
None
);
assert_eq!(
docs_region("README.md", LIFTED_FERN, LIFTED_FERN, &context),
None
);
assert_eq!(
docs_region("README.md", LIFTED_CROZIER, LIFTED_CROZIER, &context),
None
);
}
#[test]
fn the_int64_rule_takes_only_a_rounding_that_leaves_the_range() {
let fern_line = " metres=9223372036854776000,";
let crozier_line = " metres=9223372036854775807,";
let fern = [
" \"\"\"",
" Examples",
" --------",
" client.record_reading(",
fern_line,
" )",
" \"\"\"",
];
let crozier = fern.map(|line| {
if line == fern_line {
crozier_line
} else {
line
}
});
let source: crate::openapi::OpenApi = serde_json::from_value(serde_json::json!({
"openapi": "3.0.3",
"paths": {"/readings": {"post": {
"parameters": [{"name": "low", "in": "query", "schema": {
"type": "integer", "format": "int64", "example": i64::MIN
}}],
"requestBody": {"content": {"application/json": {"schema": {
"type": "object",
"properties": {
"metres": {"type": "integer", "format": "int64", "example": i64::MAX},
"trip": {"type": "integer", "format": "int64", "example": 9_007_199_254_740_993_i64},
"plain": {"type": "integer", "example": 9_223_372_036_854_000_000_i64},
"label": {"type": "string", "format": "int64", "example": 9_223_372_036_854_775_805_i64}
}
}}}},
"responses": {}
}}}
}))
.expect("a source document");
let context = Context::default().with_source_document(source);
let holds =
|rel: &str, fern_text: &[&str], crozier_text: &[&str], left: &str, right: &str| {
let pair = Pair {
rel,
fern: fern_text,
crozier: crozier_text,
context: &context,
};
int64_example_rounded_out_of_range(&pair, left, right)
};
assert!(holds(
"src/fern/client.py",
&fern,
&crozier,
fern_line,
crozier_line
));
assert!(!int64_example_rounded_out_of_range(
&pair("src/fern/client.py", &fern, &crozier),
fern_line,
crozier_line
));
let undeclared = " metres=9223372036854775806,";
let undeclared_text = fern.map(|line| if line == fern_line { undeclared } else { line });
assert!(!holds(
"src/fern/client.py",
&fern,
&undeclared_text,
fern_line,
undeclared
));
let untyped = " label=9223372036854775805,";
let untyped_fern = " label=9223372036854776000,";
let untyped_fern_text = fern.map(|line| {
if line == fern_line {
untyped_fern
} else {
line
}
});
let untyped_text = fern.map(|line| if line == fern_line { untyped } else { line });
assert!(!holds(
"src/fern/client.py",
&untyped_fern_text,
&untyped_text,
untyped_fern,
untyped
));
let near = (
" trip=9007199254740992,",
" trip=9007199254740993,",
);
let near_fern = fern.map(|line| if line == fern_line { near.0 } else { line });
let near_crozier = fern.map(|line| if line == fern_line { near.1 } else { line });
assert!(!holds(
"src/fern/client.py",
&near_fern,
&near_crozier,
near.0,
near.1
));
let low = (
" metres=-9223372036854776000,",
" metres=-9223372036854775808,",
);
let low_fern = fern.map(|line| if line == fern_line { low.0 } else { line });
let low_crozier = fern.map(|line| if line == fern_line { low.1 } else { line });
assert!(holds(
"src/fern/client.py",
&low_fern,
&low_crozier,
low.0,
low.1
));
for (left, right) in [
(" metres=9223372036854776001,", crozier_line),
(fern_line, " metres=9223372036854000000,"),
(fern_line, " other=9223372036854775807,"),
(
" metres=\"9223372036854776000\",",
" metres=\"9223372036854775807\",",
),
(fern_line, fern_line),
] {
let left_text = fern.map(|line| if line == fern_line { left } else { line });
let right_text = fern.map(|line| if line == fern_line { right } else { line });
assert!(
!holds("src/fern/client.py", &left_text, &right_text, left, right),
"{left} {right}"
);
}
assert!(!holds(
"src/fern/raw_client.py",
&fern,
&crozier,
fern_line,
crozier_line
));
let bare_fern = [fern_line];
let bare_crozier = [crozier_line];
assert!(!holds(
"src/fern/client.py",
&bare_fern,
&bare_crozier,
fern_line,
crozier_line
));
let readme_fern = ["```python", " metres=9223372036854776000,", "```"];
let readme_crozier = ["```python", " metres=9223372036854775807,", "```"];
assert!(holds(
"README.md",
&readme_fern,
&readme_crozier,
readme_fern[1],
readme_crozier[1]
));
}
#[test]
fn the_int64_rule_reads_list_elements_and_declines_other_differences() {
let source: crate::openapi::OpenApi = serde_json::from_value(serde_json::json!({
"openapi": "3.0.3",
"paths": {},
"components": {"schemas": {"Reading": {
"type": "object",
"properties": {"checkpoints": {"type": "array", "items": {
"type": "integer", "format": "int64", "example": i64::MAX
}}}
}}}
}))
.expect("a source document");
let context = Context::default().with_source_document(source);
let holds = |left: &str, right: &str| {
let fern = ["```python", left, "```"];
let crozier = ["```python", right, "```"];
let pair = Pair {
rel: "README.md",
fern: &fern,
crozier: &crozier,
context: &context,
};
int64_example_rounded_out_of_range(&pair, left, right)
};
for (left, right) in [
(
" checkpoints=[9223372036854776000],",
" checkpoints=[9223372036854775807],",
),
(" 9223372036854776000", " 9223372036854775807"),
(
" ids=[1, 9223372036854776000],",
" ids=[1, 9223372036854775807],",
),
] {
assert!(holds(left, right), "{left}");
}
for (left, right) in [
(
" name=v9223372036854776000,",
" name=v9223372036854775807,",
),
(
" metres=9223372036854776000x,",
" metres=9223372036854775807x,",
),
] {
assert!(!holds(left, right), "{left}");
}
assert_eq!(
differing_integer(" name=\"é\",", " name=\"ê\","),
None
);
assert_eq!(
differing_integer("x=12é", "x=13é"),
Some(("x=", "12", "13"))
);
}
#[test]
fn the_dollar_rule_takes_only_an_escaped_docstring_literal() {
let fern_line = " variable=\"\\\\$PS1\",";
let crozier_line = " variable=\"$PS1\",";
let fern = [
" \"\"\"",
" Examples",
" --------",
" client.define_prompt(",
fern_line,
" )",
" \"\"\"",
];
let crozier = fern.map(|line| {
if line == fern_line {
crozier_line
} else {
line
}
});
let source: crate::openapi::OpenApi = serde_json::from_value(serde_json::json!({
"openapi": "3.0.3",
"paths": {},
"components": {"schemas": {"PromptSpec": {
"type": "object",
"properties": {
"variable": {"type": "string", "const": "$PS1"},
"label": {"type": "string", "example": "$5"}
}
}}}
}))
.expect("a source document");
let context = Context::default().with_source_document(source);
let holds =
|rel: &str, fern_text: &[&str], crozier_text: &[&str], left: &str, right: &str| {
let pair = Pair {
rel,
fern: fern_text,
crozier: crozier_text,
context: &context,
};
docstring_literal_dollar_escape(&pair, left, right)
};
assert!(holds(
"src/fern/client.py",
&fern,
&crozier,
fern_line,
crozier_line
));
assert!(!docstring_literal_dollar_escape(
&pair("src/fern/client.py", &fern, &crozier),
fern_line,
crozier_line
));
let label = (" label=\"\\\\$5\",", " label=\"$5\",");
let label_fern = fern.map(|line| if line == fern_line { label.0 } else { line });
let label_crozier = fern.map(|line| if line == fern_line { label.1 } else { line });
assert!(!holds(
"src/fern/client.py",
&label_fern,
&label_crozier,
label.0,
label.1
));
for (left, right) in [
(" variable=\"\\\\$PS2\",", crozier_line),
(" variable=\"\\$PS1\",", crozier_line),
(" other=\"\\\\$PS1\",", crozier_line),
(
" variable=\"PS1\",",
" variable=\"PS1\",",
),
(
" variable=\\\\$PS1,",
" variable=$PS1,",
),
] {
let left_text = fern.map(|line| if line == fern_line { left } else { line });
let right_text = fern.map(|line| if line == fern_line { right } else { line });
assert!(
!holds("src/fern/client.py", &left_text, &right_text, left, right),
"{left} {right}"
);
}
let readme_fern = ["```python", " variable=\"\\\\$PS1\",", "```"];
let readme_crozier = ["```python", " variable=\"$PS1\",", "```"];
assert!(!holds(
"README.md",
&readme_fern,
&readme_crozier,
readme_fern[1],
readme_crozier[1]
));
let bare_fern = [fern_line];
let bare_crozier = [crozier_line];
assert!(!holds(
"src/fern/client.py",
&bare_fern,
&bare_crozier,
fern_line,
crozier_line
));
}
fn region_of(
rule: RegionRule,
rel: &str,
fern: &str,
crozier: &str,
context: &Context,
) -> Option<Region> {
let fern: Vec<&str> = fern.split('\n').collect();
let crozier: Vec<&str> = crozier.split('\n').collect();
rule(&Pair {
rel,
fern: &fern,
crozier: &crozier,
context,
})
.unwrap()
}
#[test]
fn alias_reference_correction_requires_the_actual_alias_signature_json_and_descriptions() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("docs/openapi-surface/handwritten/request-alias/fern-expected");
let fern = std::fs::read_to_string(root.join("reference.md")).unwrap();
let before = "<dl>\n<dd>\n\n**request:** `TitleAlias` \n \n</dd>\n</dl>";
let after = "<dl>\n<dd>\n\n**ticket:** `int` \n \n</dd>\n</dl>\n\n<dl>\n<dd>\n\n**caption:** `typing.Optional[str]` — Shown on cabinet labels.\n \n</dd>\n</dl>";
assert!(fern.contains(before));
let corrected = fern.replace(before, after);
let sources = python_sources(&root);
let context = Context::from_sources(
[],
sources
.iter()
.map(|(path, text)| (path.as_str(), text.as_str())),
);
let rule: RegionRule = alias_reference_parameters;
assert!(region_of(rule, "reference.md", &fern, &corrected, &context).is_some());
for wrong in [
corrected.replace("**ticket:** `int`", "**ticket:** `str`"),
corrected.replace("Shown on cabinet labels.", "Wrong description."),
corrected.replace("**caption:**", "**unknown:**"),
corrected.replace("**ticket:** `int`", "**request:** `int`"),
corrected.replace("#### 🔌 Usage", "Unexplained heading"),
corrected.replace("**request_options:**", "**extra:**"),
] {
assert!(
region_of(rule, "reference.md", &fern, &wrong, &context).is_none(),
"{wrong}"
);
}
for (from, to) in [
("json={", "data={"),
("ticket: int", "ticket: str"),
("TitleAlias = Title", "TitleAlias = str"),
("\"caption\": caption,", "\"caption\": missing,"),
] {
let changed: Vec<_> = sources
.iter()
.map(|(path, text)| (path.clone(), text.replace(from, to)))
.collect();
let context = Context::from_sources(
[],
changed
.iter()
.map(|(path, text)| (path.as_str(), text.as_str())),
);
assert!(
region_of(rule, "reference.md", &fern, &corrected, &context).is_none(),
"{from}"
);
}
assert!(region_of(rule, "client.py", &fern, &corrected, &context).is_none());
assert!(region_of(rule, "reference.md", &fern, &corrected, &Context::default()).is_none());
assert!(
reference_parameter_block(&["<dl>", "<dd>", "", "**request:** `TitleAlias` "], 0)
.is_none()
);
assert!(generated_method_parameters("unrelated text").is_empty());
}
#[test]
fn binary_json_examples_require_the_actual_bytes_json_signature_and_exact_doc_changes() {
let raw = " def ship(self, *, request: bytes, request_options=None):\n json=request,\n \"content-type\": \"application/json\",\n";
let context = Context::from_sources([], [("src/acme/vault/raw_client.py", raw)]);
let fern = " def ship(self, *, request: bytes, request_options=None):\n \"\"\"\n Examples\n --------\n client.vault.ship(request=\"string\")\n \"\"\"\n";
let corrected = fern.replace("request=\"string\"", "request=b\"string\"");
let rule: RegionRule = binary_json_body_example;
assert!(region_of(rule, "src/acme/vault/client.py", fern, &corrected, &context).is_some());
let reference = "<details><summary><code>client.vault.<a href=\"src/acme/vault/client.py\">ship</a>(...)</code></summary>\nclient.vault.ship(\n request=\"string\",\n)\n**request:** `bytes` \n</details>\n";
let correct_reference = reference.replace("request=\"string\"", "request=b\"string\"");
assert!(region_of(
rule,
"reference.md",
reference,
&correct_reference,
&context
)
.is_some());
let readme = "client.vault.ship(\n request=\"string\",\n)\n";
let correct_readme = readme.replace("request=\"string\"", "request=b\"string\"");
assert!(region_of(rule, "README.md", readme, &correct_readme, &context).is_some());
for wrong in [
corrected.replace("b\"string\"", "b\"other\""),
corrected.replace("client.vault.ship", "client.other.ship"),
corrected.replace("Examples", "Parameters"),
corrected.replace("request=b\"string\"", "extra=b\"string\""),
corrected.replace("request=b\"string\"", "request=Model(request=b\"string\")"),
] {
assert!(
region_of(rule, "src/acme/vault/client.py", fern, &wrong, &context).is_none(),
"{wrong}"
);
}
for source in [
raw.replace("request: bytes", "request: str"),
raw.replace("request: bytes", "otherrequest: bytes"),
raw.replace("json=request", "data=request"),
raw.replace("application/json", "text/plain"),
raw.replace("request: bytes", "request: str")
.replace("json=request", "request : bytes\n json=request"),
] {
let context =
Context::from_sources([], [("src/acme/vault/raw_client.py", source.as_str())]);
assert!(
region_of(rule, "src/acme/vault/client.py", fern, &corrected, &context).is_none()
);
}
assert!(region_of(
rule,
"reference.md",
reference,
&correct_reference.replace("`bytes`", "`int`"),
&context
)
.is_none());
assert!(region_of(
rule,
"README.md",
readme,
&correct_readme.replace("vault.ship", "other.ship"),
&context
)
.is_none());
assert!(region_of(
rule,
"src/acme/vault/raw_client.py",
fern,
&corrected,
&context
)
.is_none());
assert!(region_of(
rule,
"README.md",
readme,
&correct_readme,
&Context::default()
)
.is_none());
}
#[test]
fn multipart_required_file_examples_allow_only_the_proven_missing_argument() {
for (field, annotation, declaration) in [
(
"plan",
"PlanAlias",
"class Plan(Model):\n pass\nPlanAlias = Plan\n",
),
("json", "Plan", "class Plan(Model):\n pass\n"),
] {
let raw = format!(" def upload(\n self,\n ):\n data={{\n \"{field}\": json.dumps(jsonable_encoder({field})),\n }}\n files={{\n \"attachment\": attachment,\n }}\n");
let context = Context::from_sources(
[],
[
("src/acme/raw_client.py", raw.as_str()),
("src/acme/types/plan.py", declaration),
],
);
let fern = format!(" def upload(\n self,\n *,\n {field}: {annotation},\n attachment: core.File,\n ) -> None:\n \"\"\"\n Examples\n --------\n client.upload(\n {field}=Plan(bearing=1),\n )\n \"\"\"\n");
let corrected = fern.replace(
" )\n",
" attachment=\"example_attachment\",\n )\n",
);
let rule: RegionRule = multipart_object_required_file_example;
assert!(region_of(rule, "src/acme/client.py", &fern, &corrected, &context).is_some());
for changed in [
corrected.replace("example_attachment", "other_file"),
corrected.replace("bearing=1", "bearing=2"),
corrected.replace("bearing=1", "bearing=1, attachment=\"example_attachment\""),
corrected.replace("client.upload", "client.other"),
corrected.replace("attachment: core.File", "attachment: core.File = None"),
corrected.replace(" Examples", " Parameters"),
corrected.replace(" )\n", " unknown=1,\n )\n"),
] {
assert!(
region_of(rule, "src/acme/client.py", &fern, &changed, &context).is_none(),
"{changed}"
);
}
assert!(
region_of(rule, "src/acme/raw_client.py", &fern, &corrected, &context).is_none()
);
assert!(region_of(
rule,
"src/acme/client.py",
&fern,
&corrected,
&Context::default()
)
.is_none());
let not_encoded = raw.replace("json.dumps(jsonable_encoder", "convert");
let unsent = raw.replace(" \"attachment\": attachment,\n", "");
for (raw, accepted) in [(not_encoded, true), (unsent, false)] {
let context = Context::from_sources(
[],
[
("src/acme/raw_client.py", raw.as_str()),
("src/acme/types/plan.py", declaration),
],
);
assert_eq!(
region_of(rule, "src/acme/client.py", &fern, &corrected, &context).is_some(),
accepted,
"{raw}"
);
}
}
assert!(example_call_arguments(
"client.upload(value=[\"a,b\", {\"x\": (1, 2)}], attachment=\"example_attachment\")"
)
.is_some());
for invalid in ["no_call", "call(", "call(\"unterminated)", "call(x))"] {
assert!(example_call_arguments(invalid).is_none(), "{invalid}");
}
}
#[test]
fn constant_headers_are_read_from_raw_client_header_dicts() {
let context = Context::from_sources(
[],
[
(
"src/acme/beds/raw_client.py",
"json={\n \"kind\": \"x\",\n},\nheaders={\n \"X-Mist-Mode\": \"fine\",\n \"X-Vent\": str(vent) if vent is not None else None,\n},\n",
),
("src/acme/client.py", "headers={\n \"X-Other\": \"y\",\n},\n"),
],
);
assert_eq!(
context.crozier_constant_headers(),
&BTreeSet::from([("X-Mist-Mode".to_string(), "fine".to_string())])
);
assert!(Context::default().crozier_constant_headers().is_empty());
let crozier = tempfile::tempdir().unwrap();
std::fs::write(
crozier.path().join("raw_client.py"),
"headers={\n \"storage-unit\": \"GB\",\n},\n",
)
.unwrap();
let reference = tempfile::tempdir().unwrap();
assert_eq!(
Context::from_trees(reference.path(), crozier.path()).crozier_constant_headers(),
&BTreeSet::from([("storage-unit".to_string(), "GB".to_string())])
);
}
const CONSTANT_FERN: &str = "client.beds.list_beds(\n section=\"section\",\n)\n\n<dl>\n<dd>\n\n**mist_mode:** `typing.Literal` — How fine.\n \n</dd>\n</dl>\n\n<dl>\n<dd>\n\n**section:** `str` \n";
const CONSTANT_CROZIER: &str = "client.beds.list_beds(\n section=\"section\",\n)\n\n<dl>\n<dd>\n\n**section:** `str` \n";
#[test]
fn the_constant_header_docs_rule_takes_only_the_constants_reference_blocks() {
let context = Context::from_sources(
[],
[(
"raw_client.py",
"headers={\n \"X-Mist-Mode\": \"fine\",\n},\n",
)],
);
let rule: RegionRule = constant_header_docs_arguments;
assert!(region_of(
rule,
"reference.md",
CONSTANT_FERN,
CONSTANT_CROZIER,
&context
)
.is_some());
for (fern, crozier) in [
(format!("{CONSTANT_FERN}x"), format!("{CONSTANT_CROZIER}y")),
(
CONSTANT_FERN.replace("`typing.Literal`", "`str`"),
CONSTANT_CROZIER.to_string(),
),
(
CONSTANT_FERN.replace("**mist_mode:**", "**fan_speed:**"),
CONSTANT_CROZIER.to_string(),
),
(
CONSTANT_FERN.replace(
" section=\"section\",\n",
" section=\"section\",\n mist_mode=\"fine\",\n",
),
CONSTANT_CROZIER.to_string(),
),
] {
assert_eq!(
region_of(rule, "reference.md", &fern, &crozier, &context),
None,
"{fern}"
);
}
for rel in ["README.md", "src/acme/beds/client.py"] {
assert_eq!(
region_of(rule, rel, CONSTANT_FERN, CONSTANT_CROZIER, &context),
None,
"{rel}"
);
}
assert_eq!(
region_of(
rule,
"reference.md",
CONSTANT_FERN,
CONSTANT_CROZIER,
&Context::default()
),
None
);
assert_eq!(
region_of(rule, "reference.md", CONSTANT_FERN, CONSTANT_FERN, &context),
None
);
}
const NULLABLE_FERN: &str = "client.beds.list_beds(\n rows=[\n None\n ],\n)\n\n**rows:** `typing.Optional[typing.Union[typing.Optional[int], typing.Sequence[typing.Optional[int]]]]` \n";
const NULLABLE_CROZIER: &str = "client.beds.list_beds(\n rows=[\n 1\n ],\n)\n\n**rows:** `typing.Optional[typing.Union[int, typing.Sequence[int]]]` \n";
#[test]
fn the_nullable_items_rule_takes_only_the_documented_parameters() {
let context = Context::from_sources([("reference.md", NULLABLE_FERN)], []);
assert_eq!(
context.reference_nullable_items(),
&BTreeSet::from(["rows".to_string()])
);
let rule: RegionRule = nullable_items_docs;
assert!(region_of(
rule,
"reference.md",
NULLABLE_FERN,
NULLABLE_CROZIER,
&context
)
.is_some());
let call = |text: &str| text.split("\n\n").next().unwrap().to_string();
assert!(region_of(
rule,
"README.md",
&call(NULLABLE_FERN),
&call(NULLABLE_CROZIER),
&context
)
.is_some());
for (fern, crozier) in [
(
NULLABLE_FERN.replace("rows", "pots"),
NULLABLE_CROZIER.replace("rows", "pots"),
),
(
NULLABLE_FERN.to_string(),
NULLABLE_CROZIER.replace(
"typing.Union[int, typing.Sequence[int]]",
"typing.Sequence[int]",
),
),
(format!("{NULLABLE_FERN}x"), format!("{NULLABLE_CROZIER}y")),
(NULLABLE_FERN.to_string(), NULLABLE_FERN.to_string()),
] {
assert_eq!(
region_of(rule, "reference.md", &fern, &crozier, &context),
None,
"{fern}"
);
}
assert_eq!(
region_of(rule, "client.py", NULLABLE_FERN, NULLABLE_CROZIER, &context),
None
);
assert!(Context::default().reference_nullable_items().is_empty());
let reference = tempfile::tempdir().unwrap();
std::fs::write(reference.path().join("reference.md"), NULLABLE_FERN).unwrap();
let crozier = tempfile::tempdir().unwrap();
assert_eq!(
Context::from_trees(reference.path(), crozier.path()).reference_nullable_items(),
&BTreeSet::from(["rows".to_string()])
);
}
#[test]
fn the_closed_empty_object_rule_recognises_only_the_placeholder_and_its_correction() {
let holds = |fern: &str, crozier: &str| {
closed_empty_object_example_line(&pair("src/fern/client.py", &[], &[]), fern, crozier)
};
assert!(holds(
" profile={\"key\": \"value\"},",
" profile={},"
));
assert!(!holds(
" profile={\"key\": 1},",
" profile={},"
));
assert!(!holds(
" profile={\"key\": \"value\"},",
" profile={\"key\": \"value\"},"
));
assert!(!holds(
" profile={\"key\": \"value\"},",
" notes={},"
));
assert!(!holds(
" profile={\"key\": \"value\"},",
" profile={},"
));
assert!(!holds(
" 1x={\"key\": \"value\"},",
" 1x={},"
));
let fern = [
"client.firings.schedule_firing(",
" kiln=\"kiln\",",
" profile={",
" \"key\": \"value\"",
" },",
")",
"await client.firings.schedule_firing(",
" profile={",
" \"key\": \"value\"",
" },",
")",
];
let crozier = [
"client.firings.schedule_firing(",
" kiln=\"kiln\",",
" profile={},",
")",
"await client.firings.schedule_firing(",
" profile={},",
")",
];
assert_eq!(
closed_empty_object_example_region(&pair("README.md", &fern, &crozier)),
Ok(Some(Region {
fern: 2..10,
crozier: 2..6
}))
);
assert_eq!(
closed_empty_object_example_region(&pair("src/fern/client.py", &fern, &crozier)),
Ok(None)
);
let mut other = crozier;
other[3] = ");";
assert_eq!(
closed_empty_object_example_region(&pair("README.md", &fern, &other)),
Ok(None)
);
let open = [" profile={", " \"key\": \"value\"", " },"];
assert_eq!(
closed_empty_object_example_region(&pair("reference.md", &open, &open)),
Ok(None)
);
let nested = [" profile={", " \"key\": 1", " },"];
assert_eq!(
closed_empty_object_example_region(&pair(
"reference.md",
&nested,
&[" profile={},"]
)),
Ok(None)
);
}
#[test]
fn the_repeated_keyword_rule_takes_only_a_constructor_call_repeat() {
let fern = [
"client = FernApi(",
" api_key=\"<value>\",",
" api_key=\"<X-Api-Key>\",",
")",
"client.relay(",
" to=\"to\",",
")",
];
let crozier = [
"client = FernApi(",
" api_key=\"<value>\",",
")",
"client.relay(",
" to=\"to\",",
")",
];
let found =
repeated_credential_example_keyword(&pair("README.md", &fern, &crozier)).unwrap();
assert_eq!(
found,
Some(Region {
fern: 2..3,
crozier: 2..2
})
);
assert!(
repeated_credential_example_keyword(&pair("src/fern/client.py", &fern, &crozier))
.unwrap()
.is_some()
);
assert!(repeated_credential_example_keyword(&pair(
"src/fern/raw_client.py",
&fern,
&crozier
))
.unwrap()
.is_none());
assert!(
repeated_credential_example_keyword(&pair("README.md", &fern, &fern))
.unwrap()
.is_none()
);
let wrong = [
"client = FernApi(",
" api_key=\"<X-Api-Key>\",",
")",
"client.relay(",
" to=\"to\",",
")",
];
assert!(
repeated_credential_example_keyword(&pair("README.md", &fern, &wrong))
.unwrap()
.is_none()
);
let method = ["client.relay(", " to=\"a\",", " to=\"b\",", ")"];
let once = ["client.relay(", " to=\"a\",", ")"];
assert!(
repeated_credential_example_keyword(&pair("README.md", &method, &once))
.unwrap()
.is_none()
);
}
#[test]
fn the_header_rules_recognise_only_identity_header_lines() {
let lines: [&str; 0] = [];
let wrapper = pair("src/acme/core/client_wrapper.py", &lines, &lines);
assert!(sdk_identity_header_prefix(
&wrapper,
" \"X-Fern-Language\": \"Python\",",
" \"X-Crozier-Language\": \"Python\","
));
assert!(!sdk_identity_header_prefix(
&wrapper,
" \"X-Fern-Language\": \"Python\",",
" \"X-Crozier-Language\": \"Pythn\","
));
assert!(!sdk_identity_header_prefix(
&pair("client.py", &lines, &lines),
"\"X-Fern-Language\": \"Python\",",
"\"X-Crozier-Language\": \"Python\","
));
assert!(!sdk_identity_header_prefix(&wrapper, "x = 1", "x = 1"));
}
fn wrapper_pair<'a>(rel: &'a str, fern: &'a [&'a str]) -> Pair<'a> {
static PROJECT: OnceLock<Context> = OnceLock::new();
Pair {
rel,
fern,
crozier: &[],
context: PROJECT.get_or_init(|| {
Context::from_sources([], [("pyproject.toml", "[project]\nname = \"acme\"\n")])
}),
}
}
#[test]
fn the_sdk_pair_rule_takes_only_crozier_s_own_name_and_fixed_version() {
let wrapper = wrapper_pair("src/acme/core/client_wrapper.py", &[]);
assert!(sdk_name_version_header(
&wrapper,
" \"X-Fern-SDK-Version\": \"1.4.2\",",
" \"X-Crozier-SDK-Version\": \"0.0.0\","
));
assert!(sdk_name_version_header(
&wrapper,
" \"X-Fern-SDK-Name\": \"acme-sdk\",",
" \"X-Crozier-SDK-Name\": \"acme\","
));
for (fern, crozier) in [
(
"\"X-Fern-SDK-Version\": \"1.4.2\",",
"\"X-Crozier-SDK-Version\": \"9.9.9\",",
),
(
"\"X-Fern-SDK-Name\": \"a\",",
"\"X-Crozier-SDK-Name\": \"other\",",
),
(
"\"X-Fern-SDK-Name\": \"acme\",",
"\"X-Crozier-SDK-Name\": \"acme\",",
),
(
"\"X-Fern-SDK-Name\": \"a\",",
"\"X-Crozier-SDK-Version\": \"0.0.0\",",
),
(
" \"X-Fern-SDK-Name\": \"a\",",
"\"X-Crozier-SDK-Name\": \"acme\",",
),
(
"\"X-Fern-SDK-Name\": a,",
"\"X-Crozier-SDK-Name\": \"acme\",",
),
(
"\"X-Fern-SDK-Name\": \"a\"",
"\"X-Crozier-SDK-Name\": \"acme\"",
),
(
"\"X-Fern-Language\": \"a\",",
"\"X-Crozier-Language\": \"0.0.0\",",
),
] {
assert!(
!sdk_name_version_header(&wrapper, fern, crozier),
"{crozier}"
);
}
let lines: [&str; 0] = [];
assert!(!sdk_name_version_header(
&pair("core/client_wrapper.py", &lines, &lines),
"\"X-Fern-SDK-Name\": \"a\",",
"\"X-Crozier-SDK-Name\": \"acme\","
));
let without = ["headers = {", "}"];
let added = wrapper_pair("core/client_wrapper.py", &without);
assert!(sdk_name_version_header_added(
&added,
" \"X-Crozier-SDK-Name\": \"acme\","
));
assert!(sdk_name_version_header_added(
&added,
" \"X-Crozier-SDK-Version\": \"0.0.0\","
));
for crozier in [
" \"X-Crozier-SDK-Name\": \"other\",",
" \"X-Crozier-SDK-Version\": \"0.0.1\",",
" \"X-Crozier-Language\": \"Python\",",
] {
assert!(!sdk_name_version_header_added(&added, crozier), "{crozier}");
}
let with = [" \"X-Fern-SDK-Name\": \"acme\","];
assert!(!sdk_name_version_header_added(
&wrapper_pair("core/client_wrapper.py", &with),
" \"X-Crozier-SDK-Version\": \"0.0.0\","
));
assert!(!sdk_name_version_header_added(
&wrapper_pair("client.py", &without),
" \"X-Crozier-SDK-Name\": \"acme\","
));
}
fn lines(text: &str) -> Vec<&str> {
text.split('\n').collect()
}
fn metadata_region(fern: &str, crozier: &str, rel: &str) -> Option<Region> {
let (fern, crozier) = (lines(fern), lines(crozier));
metadata_generator_config(&pair(rel, &fern, &crozier)).unwrap()
}
#[test]
fn the_metadata_rule_takes_only_generator_config_differences() {
let record = crate::emit::FERN_METADATA_RECORD;
let region = |fern: &str| metadata_region(fern, record, FERN_METADATA);
let literals = record.replace("python_enums", "literals");
assert_eq!(
region(&literals),
Some(Region {
fern: 6..7,
crozier: 6..7
})
);
let braced = record.replace("python_enums", "lit}{\\\"erals");
assert!(region(&braced).is_some(), "{braced}");
let start = record.find(" \"generatorConfig\"").unwrap();
let end = record.find(" \"invokedBy\"").unwrap();
let absent = format!("{}{}", &record[..start], &record[end..]);
assert_eq!(
region(&absent),
Some(Region {
fern: 4..4,
crozier: 4..9
})
);
assert_eq!(region(&literals.replace("github", "gitlab")), None);
assert_eq!(region(&braced.replace("github", "gitlab")), None);
assert_eq!(
region(&literals.replace("\"ci\",", "\"ci\" ,")),
None,
"a formatting change outside the member"
);
assert_eq!(
metadata_region(&literals, record, "types/user_metadata.json"),
None
);
assert_eq!(region(record), None);
let sdk = crate::emit::certified_fern_pair().python_sdk;
let shared = literals.replace(
&format!("\"generatorVersion\": \"{sdk}\",\n \"generatorConfig\""),
&format!("\"generatorVersion\": \"{sdk}\", \"generatorConfig\""),
);
assert_ne!(shared, literals);
assert_eq!(region(&shared), None);
assert_eq!(region("{ not json"), None);
}
#[test]
fn the_metadata_rule_takes_only_crozier_s_fixed_record() {
let record = crate::emit::FERN_METADATA_RECORD;
let fern = record.replace("python_enums", "literals");
for crozier in [
record.replace("python_enums", "literals_too"),
record.replace("python_enums", "literals"),
record.replace("\"ci\"", "\"manual\""),
] {
assert_eq!(metadata_region(&fern, &crozier, FERN_METADATA), None);
}
}
#[test]
fn the_member_is_located_string_aware() {
assert_eq!(
generator_config_member("{\n \"a\": \"}{\",\n \"generatorConfig\": {\n \"x\": \"}\"\n },\n \"b\": 1\n}"),
Member::Lines(2..5)
);
assert_eq!(
generator_config_member("{\n \"generatorConfig\": \"x,}\"\n}"),
Member::Lines(1..2)
);
assert_eq!(
generator_config_member("{\n \"generatorConfig\": [1, {\"a\": 2}],\n \"b\": 1\n}"),
Member::Lines(1..2)
);
assert_eq!(
generator_config_member("{\n \"generatorConfig\": 12\n}"),
Member::Lines(1..2)
);
assert_eq!(
generator_config_member("{\"a\": 1, \"generatorConfig\": {}}"),
Member::Shared
);
assert_eq!(
generator_config_member(
"{\n \"x\": {\"generatorConfig\": {}},\n \"y\": \"generatorConfig\"\n}"
),
Member::Absent
);
assert_eq!(generator_config_member("{\"a\": \"open"), Member::Absent);
assert_eq!(
generator_config_member("{\"generatorConfig\": \"open"),
Member::Absent
);
assert_eq!(
generator_config_member("{\"generatorConfig\": }"),
Member::Absent
);
}
#[test]
fn the_import_order_rule_takes_only_a_reordered_type_checking_block() {
let crozier = lines(
concat!("import typing\n\nif typing.TYPE_CHECKING:\n from .a import A\n from .b import B\n", "_dynamic_imports = {}\n"),
);
let region = |fern: &str, rel: &str| {
let fern = lines(fern);
init_type_checking_import_order(&pair(rel, &fern, &crozier)).unwrap()
};
let reordered = concat!(
"import typing\n\nif typing.TYPE_CHECKING:\n from .b import B\n ",
"from .a import A\n_dynamic_imports = {}\n"
);
assert_eq!(
region(reordered, "src/acme/__init__.py"),
Some(Region {
fern: 3..5,
crozier: 3..5
})
);
assert!(region(reordered, "__init__.py").is_some());
let other = reordered.replace("from .b import B", "from .c import C");
assert_eq!(region(&other, "__init__.py"), None);
assert_eq!(region(reordered, "client.py"), None);
assert_eq!(region("import typing\n", "__init__.py"), None);
assert_eq!(region(&crozier.join("\n"), "__init__.py"), None);
let shallow = "if typing.TYPE_CHECKING:\n from .b import B\n from .a import A\nx\n";
assert_eq!(region(shallow, "__init__.py"), None);
}
#[test]
fn ruff_refusing_the_input_surfaces_as_an_error() {
let error = ruff_isort("def (:\n").unwrap_err();
assert!(error.contains("ruff isort failed"), "{error}");
assert_eq!(
ruff_isort("import typing\nimport enum\n").unwrap(),
"import enum\nimport typing\n"
);
}
#[test]
fn tokens_split_identifiers_from_everything_between_them() {
assert_eq!(
tokens("from a.b import C, déjà_2"),
["from", " ", "a", ".", "b", " ", "import", " ", "C", ",", " ", "déjà_2"]
);
assert_eq!(tokens("(x)"), ["(", "x", ")"]);
assert!(tokens("").is_empty());
}
#[test]
fn the_differing_window_drops_common_leading_and_trailing_lines() {
assert_eq!(differing_window(&["a", "b"], &["a", "b"]), None);
assert_eq!(
differing_window(&["a", "x", "c"], &["a", "y", "z", "c"]),
Some(Region {
fern: 1..2,
crozier: 1..3
})
);
assert_eq!(
differing_window(&["a"], &["a", "a"]),
Some(Region {
fern: 1..1,
crozier: 1..2
})
);
}
#[test]
fn every_kind_has_a_spelling_and_a_meaning() {
let spellings: BTreeSet<&str> = Kind::ALL.iter().map(|kind| kind.as_str()).collect();
assert_eq!(spellings.len(), Kind::ALL.len());
for kind in Kind::ALL {
assert!(!kind.meaning().is_empty());
let parsed: Kind = crate::yaml::from_str(kind.as_str()).unwrap();
assert_eq!(parsed, kind);
}
}
#[test]
fn body_query_departure_accepts_only_the_body_variable_substitution() {
let fern = [
" def record(self, *, level: int, reading_level: int):",
" params={",
" \"level\": level,",
" },",
" json={",
" \"level\": level,",
" },",
];
let mut crozier = fern;
crozier[5] = " \"level\": reading_level,";
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &crozier)),
Ok(Some(super::Region {
fern: 5..6,
crozier: 5..6
}))
);
assert_eq!(
super::body_query_parameter_value(&pair("src/api/client.py", &fern, &crozier)),
Ok(None)
);
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &fern)),
Ok(None)
);
for changed in [
" \"level\": other_level,",
" \"threshold\": reading_level,",
" \"level\": 3,",
" \"level\": \"a b\",",
] {
crozier[5] = changed;
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &crozier)),
Ok(None)
);
}
crozier[5] = " \"level\": reading_level,";
crozier[0] = " def record(self, *, level: int):";
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &crozier)),
Ok(None)
);
crozier[0] = fern[0];
crozier[2] = " \"threshold\": level,";
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &crozier)),
Ok(None)
);
}
#[test]
fn body_query_departure_keeps_converter_annotation_and_string_contents() {
let fern = [
" def record(self, *, level: int, reading_level: int):",
" params={",
" \"level\": level,",
" },",
" json={",
" \"level\": convert_and_respect_annotation_metadata(object_=level, annotation=int, direction=\"write\"),",
" },",
];
let crozier = [
fern[0],
fern[1],
fern[2],
fern[3],
fern[4],
" \"level\": convert_and_respect_annotation_metadata(",
" object_=reading_level, annotation=int, direction=\"write\",",
" ),",
fern[6],
];
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &crozier)),
Ok(Some(super::Region {
fern: 5..6,
crozier: 5..8
}))
);
assert!(matches!(
super::body_query_parameter_value(&pair("raw_client.py", &fern, &crozier)),
Ok(Some(_))
));
let mut changed = crozier;
changed[6] =
" object_=reading_level, annotation=str, direction=\"write\"";
assert_eq!(
super::body_query_parameter_value(&pair("src/api/raw_client.py", &fern, &changed)),
Ok(None)
);
assert_eq!(
super::compact_python(&[" \"a b\" : '\\'c d' "]),
"\"a b\":'\\'c d'"
);
}
fn source_context(document: serde_json::Value, crozier: &[(&str, &str)]) -> Context {
Context::from_sources([], crozier.iter().copied())
.with_source_document(serde_json::from_value(document).expect("a source document"))
}
fn line_holds(rule: LineRule, rel: &str, context: &Context, fern: &str, crozier: &str) -> bool {
let (fern_lines, crozier_lines) = ([fern], [crozier]);
rule(
&Pair {
rel,
fern: &fern_lines,
crozier: &crozier_lines,
context,
},
fern,
crozier,
)
}
fn added_at_holds(rule: AddedAtRule, rel: &str, context: &Context, crozier: &str) -> Vec<bool> {
let lines: Vec<&str> = crozier.split('\n').collect();
(0..lines.len())
.map(|at| {
rule(
&Pair {
rel,
fern: &[],
crozier: &lines,
context,
},
at,
)
})
.collect()
}
#[test]
fn anyof_path_param_schema_example_admits_only_the_declared_composition_example() {
let document = |schema: serde_json::Value, location: &str| {
serde_json::json!({"openapi": "3.0.3", "paths": {"/docks/{dock}": {"get": {
"parameters": [{"name": "dock", "in": location, "required": true, "schema": schema}],
"responses": {}
}}}})
};
let composed = serde_json::json!({
"example": "northbasin",
"anyOf": [{"type": "string", "pattern": "^[a-z]+$"}, {"type": "string", "format": "uuid"}]
});
let context = source_context(document(composed.clone(), "path"), &[]);
let (fern, crozier) = (" dock=\"dock\",", " dock=\"northbasin\",");
let rule: LineRule = anyof_path_param_schema_example;
for rel in ["README.md", "reference.md", "src/fern/docks/client.py"] {
assert!(line_holds(rule, rel, &context, fern, crozier), "{rel}");
}
assert!(!line_holds(
rule,
"src/fern/docks/raw_client.py",
&context,
fern,
crozier
));
for (left, right) in [
(fern, " dock=\"southbasin\","),
(fern, " berth=\"northbasin\","),
(fern, " dock=\"northbasin\","),
(" dock=\"other\",", crozier),
(fern, fern),
] {
assert!(
!line_holds(rule, "README.md", &context, left, right),
"{right}"
);
}
let query = source_context(document(composed, "query"), &[]);
assert!(!line_holds(rule, "README.md", &query, fern, crozier));
let plain = source_context(
document(
serde_json::json!({"type": "string", "example": "northbasin"}),
"path",
),
&[],
);
assert!(!line_holds(rule, "README.md", &plain, fern, crozier));
assert!(!line_holds(
rule,
"README.md",
&Context::default(),
fern,
crozier
));
}
#[test]
fn text_response_optional_query_examples_admit_only_a_declared_optional_query_value() {
let document = |media: &str, required: bool| {
serde_json::json!({"openapi": "3.0.3", "paths": {"/log": {"get": {
"parameters": [{"name": "columns", "in": "query", "required": required,
"schema": {"type": "string", "example": "vessel,arrival"}}],
"responses": {"200": {"description": "ok", "content": {media: {"schema": {"type": "string"}}}}}
}}}})
};
let context = source_context(document("text/plain", false), &[]);
let line = " columns=\"vessel,arrival\",";
for rel in ["README.md", "reference.md", "src/fern/docks/client.py"] {
assert!(
text_response_optional_query_examples(&pair_with(rel, &context), line),
"{rel}"
);
}
assert!(!text_response_optional_query_examples(
&pair_with("README.md", &context),
" columns=\"vessel\","
));
assert!(!text_response_optional_query_examples(
&pair_with("README.md", &context),
" rows=\"vessel,arrival\","
));
assert!(!text_response_optional_query_examples(
&pair_with("src/fern/docks/raw_client.py", &context),
line
));
for (media, required) in [("application/json", false), ("text/plain", true)] {
let other = source_context(document(media, required), &[]);
assert!(
!text_response_optional_query_examples(&pair_with("README.md", &other), line),
"{media} {required}"
);
}
assert!(!text_response_optional_query_examples(
&pair_with("README.md", &Context::default()),
line
));
}
fn pair_with<'a>(rel: &'a str, context: &'a Context) -> Pair<'a> {
Pair {
rel,
fern: &[],
crozier: &[],
context,
}
}
#[test]
fn optional_lowered_required_query_array_example_admits_only_the_declared_list() {
let document = |required: bool, example: serde_json::Value| {
serde_json::json!({"openapi": "3.0.3", "paths": {"/tides": {"get": {
"parameters": [{"name": "gauges", "in": "query", "required": required,
"schema": {"type": "array", "items": {"type": "number"}, "example": example}}],
"responses": {}
}}}})
};
let context = source_context(document(true, serde_json::json!([3, 7])), &[]);
let rule: AddedAtRule = optional_lowered_required_query_array_example;
let wrapped = "client.tides.get_surge(\n station_id=\"s\",\n gauges=[\n 3,\n 7\n ],\n)";
assert_eq!(
added_at_holds(rule, "reference.md", &context, wrapped),
[false, false, true, true, true, true, false]
);
let flat = " client.tides.get_surge(\n gauges=[3.0, 7.0],\n )";
assert_eq!(
added_at_holds(rule, "src/fern/tides/client.py", &context, flat),
[false, true, false]
);
for (rel, text) in [
(
"reference.md",
wrapped.replace(" 7\n", " 8\n"),
),
(
"reference.md",
wrapped.replace(" 7\n", " 7\n"),
),
("src/fern/tides/client.py", flat.replace("7.0", "7.5")),
(
"src/fern/tides/client.py",
flat.replace("[3.0, 7.0]", "[1.1]"),
),
("src/fern/tides/raw_client.py", flat.to_string()),
] {
assert!(
!added_at_holds(rule, rel, &context, &text)
.iter()
.any(|holds| *holds),
"{text}"
);
}
for other in [
source_context(document(false, serde_json::json!([3, 7])), &[]),
source_context(document(true, serde_json::Value::Null), &[]),
Context::default(),
] {
assert!(!added_at_holds(rule, "reference.md", &other, wrapped)
.iter()
.any(|holds| *holds));
}
assert!(item_is_value("\"a\"", &serde_json::json!("a")));
assert!(item_is_value("True", &serde_json::json!(true)));
assert!(!item_is_value("{}", &serde_json::json!({})));
}
#[test]
fn collapsed_anyof_property_description_admits_only_the_property_description() {
let property = |first: serde_json::Value| {
serde_json::json!({"openapi": "3.0.3", "paths": {}, "components": {"schemas": {
"Charter": {"type": "object", "required": ["skipper"], "properties": {
"skipper": {"description": "The skipper's ID.", "anyOf": [first, {"type": "string"}]}
}}
}}})
};
let context = source_context(
property(serde_json::json!({"type": "string", "format": "uuid"})),
&[],
);
let rule: LineRule = collapsed_anyof_property_description;
let row = "**skipper:** `str` ";
assert!(line_holds(
rule,
"reference.md",
&context,
row,
"**skipper:** `str` — The skipper's ID."
));
assert!(!line_holds(
rule,
"reference.md",
&context,
row,
"**skipper:** `str` — Another."
));
assert!(!line_holds(
rule,
"reference.md",
&context,
"**guests:** `int` ",
"**guests:** `int` — The skipper's ID."
));
let field = " skipper: str";
assert!(line_holds(
rule,
"src/fern/types/charter.py",
&context,
field,
" skipper: str = pydantic.Field()"
));
assert!(!line_holds(
rule,
"src/fern/types/charter.py",
&context,
field,
" skipper: str = pydantic.Field(alias=\"s\")"
));
assert!(!line_holds(
rule,
"src/fern/types/charter.py",
&context,
" guests: int",
" guests: int = pydantic.Field()"
));
let added: AddedAtRule = collapsed_anyof_property_description_line;
let model = "class Charter(UniversalBaseModel):\n skipper: str = pydantic.Field()\n \"\"\"\n The skipper's ID.\n \"\"\"\n\n vessel: str";
assert_eq!(
added_at_holds(added, "src/fern/types/charter.py", &context, model),
[false, false, true, true, true, true, false]
);
let docstring = " Parameters\n ----------\n skipper : str\n The skipper's ID.\n\n guests : int";
assert_eq!(
added_at_holds(added, "src/fern/charters/client.py", &context, docstring),
[false, false, false, true, false, false]
);
for text in [
model.replace("The skipper's ID.", "Another."),
docstring.replace("The skipper's ID.", "Another."),
] {
assert!(
!added_at_holds(added, "src/fern/charters/client.py", &context, &text)
.iter()
.any(|holds| *holds)
);
}
assert!(!added_at_holds(added, "reference.md", &context, docstring)
.iter()
.any(|holds| *holds));
let described = source_context(
property(serde_json::json!({"type": "string", "description": "Own."})),
&[],
);
assert!(!line_holds(
rule,
"reference.md",
&described,
row,
"**skipper:** `str` — The skipper's ID."
));
assert!(
!added_at_holds(added, "src/fern/charters/client.py", &described, docstring)
.iter()
.any(|holds| *holds)
);
assert!(!line_holds(
rule,
"reference.md",
&Context::default(),
row,
"**skipper:** `str` — The skipper's ID."
));
}
#[test]
fn emoji_tag_reference_identifiers_admit_only_the_raw_tag_spelling_of_a_module() {
let document = serde_json::json!({"openapi": "3.0.3", "paths": {"/legacy": {"get": {
"tags": ["🚧 Legacy endpoints"], "responses": {}
}}}});
let modules = [("src/fern/legacy_endpoints/client.py", "")];
let context = source_context(document.clone(), &modules);
let rule: LineRule = emoji_tag_reference_identifiers;
let crozier = "<code>client.legacy_endpoints.<a href=\"src/fern/legacy_endpoints/client.py\">list</a>(...)</code>";
let fern = crozier.replace("legacy_endpoints", "🚧_legacy_endpoints");
for rel in ["README.md", "reference.md"] {
assert!(line_holds(rule, rel, &context, &fern, crozier), "{rel}");
}
assert!(!line_holds(
rule,
"src/fern/client.py",
&context,
&fern,
crozier
));
let half = crozier.replacen("legacy_endpoints", "🚧_legacy_endpoints", 1);
assert!(!line_holds(rule, "reference.md", &context, &fern, &half));
let other = fern.replace("🚧_legacy_endpoints.<a", "🚧_legacy.<a");
assert!(!line_holds(rule, "reference.md", &context, &other, crozier));
let without_module = source_context(document, &[("src/fern/other/client.py", "")]);
assert!(!line_holds(
rule,
"reference.md",
&without_module,
&fern,
crozier
));
assert!(!line_holds(
rule,
"reference.md",
&Context::default(),
&fern,
crozier
));
assert!(!line_holds(
rule,
"reference.md",
&context,
crozier,
crozier
));
}
#[test]
fn a_list_of_files_takes_the_list_placeholder() {
let raw = " def deploy(\n self,\n ):\n files={\n \"bundle\": bundle,\n }\n";
let context = Context::from_sources([], [("src/acme/raw_client.py", raw)]);
let fern = " def deploy(\n self,\n *,\n bundle: typing.List[core.File],\n ) -> None:\n \"\"\"\n Examples\n --------\n client.deploy(\n slug=\"s\",\n )\n \"\"\"\n";
let rule: RegionRule = multipart_object_required_file_example;
let listed = fern.replace(
" slug=\"s\",\n",
" slug=\"s\",\n bundle=[\"example_bundle\"],\n",
);
assert!(region_of(rule, "src/acme/client.py", fern, &listed, &context).is_some());
let single = fern.replace(
" slug=\"s\",\n",
" slug=\"s\",\n bundle=\"example_bundle\",\n",
);
assert!(region_of(rule, "src/acme/client.py", fern, &single, &context).is_none());
}
}