//! First-class typed operators — the `op:` / `with:` third step arm.
//!
//! An operator is a *packaged, typed* unit of work configured (not scripted) via a
//! `with:` block. Unlike an opaque `command:` shell string, an operator declares its
//! I/O to the **same** [`crate::asset::AssetGraph`] the SQL path uses, so lineage,
//! ordering, and stale-propagation hold at its boundary — a step that isn't in the
//! graph can silently ship a stale artifact on a green run (the exact defect the
//! `edgar_gleif` `package` step patches by hand today).
//!
//! Operators are addressed by `op: <name>@<semver-req>` and resolved from a built-in
//! [`catalog`] — a namespace deliberately *distinct* from the pipeline `registry`.
//! Config is validated by typed deserialization (a `with:`
//! block that doesn't deserialize into the operator's config is a load-time error);
//! JSON-Schema emission for Brightfield authoring forms is a later addition.
use std::collections::HashMap;
// `BTreeMap` — not `HashMap` — wherever the map's iteration order reaches an output:
// the fetch operators' `headers`, and `parquet_export`'s `metadata`, whose order is
// written verbatim into the Parquet footer and so into the file's hash.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use serde::Deserialize;
use serde_yaml::Value;
use crate::engine::StepOutput;
use crate::error::{Error, Result};
mod ducklake_publish;
/// Assets an operator step reads and produces — merged into the [`crate::asset::AssetGraph`]
/// exactly as SQL introspection and command `produces`/`depends_on` are.
///
/// **Declared case, not lowercased.** `asset.rs` is the one place that normalizes a
/// name for graph identity (a DuckDB table identifier is case-insensitive, so two
/// spellings of the same table have to land on one node) — an operator returning an
/// already-lowercased name here would throw away the real, possibly mixed on-disk case
/// before anything downstream ever saw it, which is exactly how a produced FILE's true
/// spelling used to go missing. Return exactly what the config said.
#[derive(Debug, Clone, Default)]
pub struct OpAssets {
pub produces: Vec<String>,
pub reads: Vec<String>,
/// What each name in `produces`/`reads` actually is, set by the same operator
/// that declares the name — the one place that already knows, since it is the
/// operator's own typed config (`dest:`, `input:`, `members:`, …) that says so,
/// not a guess from the string or a filesystem probe made later. Populated via
/// `record_produces`/`record_reads` so a name can never land in `produces`/
/// `reads` without a kind alongside it.
pub kinds: BTreeMap<String, crate::asset_kind::AssetKind>,
}
impl OpAssets {
/// Record a produced asset with its kind, keeping `produces` and `kinds` from
/// ever drifting apart.
pub fn record_produces(&mut self, name: impl Into<String>, kind: crate::asset_kind::AssetKind) {
let name = name.into();
self.kinds.insert(name.clone(), kind);
self.produces.push(name);
}
/// Record a read asset with its kind, keeping `reads` and `kinds` from ever
/// drifting apart.
pub fn record_reads(&mut self, name: impl Into<String>, kind: crate::asset_kind::AssetKind) {
let name = name.into();
self.kinds.insert(name.clone(), kind);
self.reads.push(name);
}
}
/// Execution context handed to an operator's [`Operator::run`].
pub struct OpContext<'a> {
/// Manifest directory — resolve relative paths (`dest`, `out`) against this.
pub dir: &'a Path,
/// The pipeline's DuckDB file — operators that touch tables open this.
pub db_path: &'a Path,
/// `ARC_PARAM_*` environment for the step, passed through to subprocess operators.
pub env: &'a HashMap<String, String>,
/// Step timeout, if any — enforced by `run_process` (Inherit mode).
pub timeout: Option<Duration>,
/// The shared fetch cache, when the environment gives the run one. Ingress
/// operators consult it so the same URL is transferred once however many
/// Protocols want it; `None` is a run with no cache, which every other operator
/// is anyway.
pub cache: Option<&'a crate::fetch_cache::FetchCache>,
}
/// A first-class typed operator. Registered in the [`catalog`], addressed by
/// `op: <name>@<semver>`.
pub trait Operator: Sync {
/// Catalog name — the `<name>` in `op: <name>@<ver>`.
fn name(&self) -> &'static str;
/// Operator version. A Protocol pins it with a semver constraint, exactly as
/// `engine_version` pins the engine — extending local≡cloud parity to every operator.
fn version(&self) -> semver::Version;
/// Validate the config and declare the assets this step reads/produces, so lineage
/// holds at the boundary. Doubles as the config-validation gate (a `with:` that does
/// not deserialize is an `Err` here, surfaced at manifest load).
fn assets(&self, with: &Value) -> Result<OpAssets>;
/// Execute the operator.
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput>;
}
static PARQUET_EXPORT: ParquetExport = ParquetExport;
#[cfg(feature = "http-fetch")]
static HTTP_FETCH: HttpFetch = HttpFetch;
#[cfg(feature = "http-fetch")]
static HTML_LINK_DISCOVER: HtmlLinkDiscover = HtmlLinkDiscover;
static ARCHIVE_EXTRACT: ArchiveExtract = ArchiveExtract;
static DATAPACKAGE_DESCRIBE: DatapackageDescribe = DatapackageDescribe;
static FINETYPE_VALIDATE: FinetypeValidate = FinetypeValidate;
static SPLINK_RESOLVE: SplinkResolve = SplinkResolve;
static GLEIF_RA_FETCH: GleifRaFetch = GleifRaFetch;
static UMAP_PROJECT: UmapProject = UmapProject;
static TEXT_EMBED: TextEmbed = TextEmbed;
static UV: Uv = Uv;
static DUCKLAKE_PUBLISH: ducklake_publish::DucklakePublish = ducklake_publish::DucklakePublish;
#[cfg(feature = "opendal")]
static OPENDAL_FETCH: OpendalFetch = OpendalFetch;
/// The built-in operator catalog. Two entries are feature-gated: the ureq-backed
/// ingress ops (`http_fetch`, `html_link_discover`) appear only under `http-fetch`
/// (enabled via `cli`; a `default-features = false` library consumer builds without
/// them and without ureq — see Cargo.toml), and `opendal_fetch` only under `opendal`
/// (off by default to keep the single binary lean).
fn catalog() -> Vec<&'static dyn Operator> {
#[allow(unused_mut)]
let mut ops: Vec<&'static dyn Operator> = vec![
&PARQUET_EXPORT,
&ARCHIVE_EXTRACT,
&DATAPACKAGE_DESCRIBE,
&FINETYPE_VALIDATE,
&SPLINK_RESOLVE,
&GLEIF_RA_FETCH,
&UMAP_PROJECT,
&TEXT_EMBED,
&UV,
&DUCKLAKE_PUBLISH,
];
#[cfg(feature = "http-fetch")]
ops.extend([
&HTTP_FETCH as &'static dyn Operator,
&HTML_LINK_DISCOVER as &'static dyn Operator,
]);
#[cfg(feature = "opendal")]
ops.push(&OPENDAL_FETCH);
ops
}
/// Resolve an `op:` reference (`name` or `name@<semver-req>`) from the catalog,
/// checking the version constraint. Returns a validation error if the operator is
/// unknown or the installed version doesn't satisfy the constraint.
pub fn resolve(op_ref: &str) -> Result<&'static dyn Operator> {
let (name, req) = match op_ref.split_once('@') {
Some((n, r)) => (n.trim(), Some(r.trim())),
None => (op_ref.trim(), None),
};
let op = catalog()
.into_iter()
.find(|o| o.name() == name)
.ok_or_else(|| {
Error::ManifestValidation(format!(
"unknown operator '{}' (not in the operator catalog)",
name
))
})?;
if let Some(req) = req {
let vreq = semver::VersionReq::parse(req).map_err(|_| {
Error::ManifestValidation(format!(
"operator '{}': invalid version constraint '{}' (expected semver, e.g. '1' or '^1.2')",
name, req
))
})?;
if !vreq.matches(&op.version()) {
return Err(Error::ManifestValidation(format!(
"operator '{}@{}' not satisfied by installed version {}",
name,
req,
op.version()
)));
}
}
Ok(op)
}
/// Resolve and declare an operator step's assets in one call — used by manifest
/// validation and the asset-graph builder (which have no [`OpContext`]).
pub fn assets_for(op_ref: &str, with: Option<&Value>) -> Result<OpAssets> {
resolve(op_ref)?.assets(with.unwrap_or(&Value::Null))
}
/// Every operator in the built-in [`catalog`], by name, in catalog order.
///
/// `opendal_fetch` appears only when the `opendal` feature is on (it is only in the
/// catalog then). Used by the `arc mcp` `operator_describe` tool to enumerate the
/// operators an authoring UI can pick from, and by the tests that walk every one.
#[cfg(any(feature = "mcp", test))]
pub(crate) fn catalog_names() -> Vec<&'static str> {
catalog().into_iter().map(|o| o.name()).collect()
}
/// The JSON Schema (Draft 2020-12) for an operator's `with:` block, or `None` for a
/// name not in the catalog.
///
/// This is the authoring-form emission the module doc anticipates: it describes the
/// shape a `with:` block must take so an editor or the `arc mcp` `operator_describe`
/// tool can build a form or validate a draft. It is a *description*, not the gate —
/// each operator's typed `serde` deserialize (see its `…Config::parse`) remains the
/// load-time validator, and this schema is hand-kept in step with it (the
/// `every_catalog_operator_has_a_with_schema` test fails if a new operator is added
/// without one). [`description`] reads it too, for each role's `required` and
/// `min_items` and for the `parameters` left once the roles are taken out.
///
/// It answers for a name whatever the build holds: `opendal_fetch` has a schema in a
/// build without `opendal`. Whether the build holds an operator is [`catalog`]'s
/// answer.
pub(crate) fn with_schema(op_name: &str) -> Option<serde_json::Value> {
use serde_json::json;
// A `with:` schema: an object of the operator's typed fields, closed to unknown
// keys (every config is `#[serde(deny_unknown_fields)]`).
fn object(properties: serde_json::Value, required: &[&str]) -> serde_json::Value {
json!({
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"additionalProperties": false,
"properties": properties,
"required": required,
})
}
let str_headers = json!({
"type": "object",
"description": "Extra request headers (a default User-Agent is set unless overridden).",
"additionalProperties": { "type": "string" }
});
let schema = match op_name {
"parquet_export" => object(
json!({
"input": { "type": "string", "description": "Source table or view in the pipeline DB to export." },
"dest": { "type": "string", "description": "Destination Parquet path, relative to the protocol directory." },
"compression": { "type": "string", "description": "Parquet codec.", "default": "zstd" },
"row_group_size": { "type": "integer", "minimum": 0, "description": "Optional Parquet row-group size." },
"order_by": { "type": "string", "description": "Optional ORDER BY clause applied to the export." },
"metadata": {
"type": "object",
"additionalProperties": { "type": "string" },
"description": "Key-value metadata stamped into the Parquet footer. Keys and values are strings, written as UTF-8 bytes; read them back with decode(key)/decode(value) over parquet_kv_metadata(), since Parquet stores them untyped and DuckDB returns BLOB. Entries are emitted in sorted key order. Declaring any metadata changes the file's bytes and so its hash; declaring none leaves the output unchanged."
}
}),
&["input", "dest"],
),
"http_fetch" => object(
json!({
"url": { "type": "string", "description": "Source URL (http/https)." },
"out": { "type": "string", "description": "Destination path, relative to the protocol directory. Written atomically." },
"headers": str_headers,
"sha256": { "type": "string", "pattern": "^[0-9a-fA-F]{64}$", "description": "Pin the artifact's SHA-256. Pinned bytes are taken from the shared fetch cache without asking the origin, and a transfer that does not match is refused." }
}),
&["url", "out"],
),
"opendal_fetch" => object(
json!({
"from": { "type": "string", "description": "Scheme-dispatched source: https://host/key, s3://bucket/key, …" },
"to": { "type": "string", "description": "Destination path, relative to the protocol directory. Written atomically." },
"headers": str_headers
}),
&["from", "to"],
),
"html_link_discover" => object(
json!({
"url": { "type": "string", "description": "Index page to fetch (http/https)." },
"pattern": { "type": "string", "description": "Regex tested (unanchored) against each raw href; matches are kept." },
"out": { "type": "string", "description": "Destination URL-list path (newline-delimited), relative to the protocol dir." },
"headers": str_headers
}),
&["url", "pattern", "out"],
),
"archive_extract" => object(
json!({
"archive": { "type": "string", "description": "Source .zip, relative to the protocol dir." },
"members": {
"type": "array",
"items": { "type": "string" },
"description": "Explicit member names to extract (exact match). Supply this or `pattern`."
},
"pattern": { "type": "string", "description": "Regex tested (unanchored) against each member name; matches are extracted. Supply this or `members`." },
"dest": { "type": "string", "description": "Destination directory, relative to the protocol dir." }
}),
&["archive", "dest"],
),
"datapackage_describe" => object(
json!({
"parquet": { "type": "string", "description": "Built Parquet whose columns FineType types." },
"overrides": { "type": "string", "description": "Curated descriptor sidecar (JSON) overlaid onto FineType's base." },
"out": { "type": "string", "description": "datapackage.json to write." },
"expect_finetype_version": { "type": "string", "description": "Pin the run to one exact finetype release. Refuses, naming both versions, unless the resolved `finetype` on PATH reports exactly this version." },
"nominations": { "type": "string", "description": "Declared column types (conventionally `nominations.finetype.json`) passed to `finetype profile --nominations`. Needs a finetype that carries the flag; the step refuses before profiling when the one on PATH is older." }
}),
&["parquet", "overrides", "out"],
),
"finetype_validate" => object(
json!({
"parquet": { "type": "string", "description": "Built Parquet to check." },
"schema": { "type": "string", "description": "Self-derived JSON-Schema contract to check against." },
"extension": { "type": "string", "description": "Optional per-step override for the FineType DuckDB extension path (else FINETYPE_DUCKDB_EXT)." }
}),
&["parquet", "schema"],
),
"splink_resolve" => object(
json!({
"edgar": { "type": "string", "description": "EDGAR/SEC-entity Parquet (the crosswalk's left side)." },
"gleif": { "type": "string", "description": "GLEIF golden-copy Parquet (the right side)." },
"out": { "type": "string", "description": "Resolved-crosswalk Parquet to write." },
"sample": { "type": "integer", "minimum": 0, "default": 0, "description": "Optional GLEIF-row cap for a fast smoke test; 0 resolves the full corpus." }
}),
&["edgar", "gleif", "out"],
),
"gleif_ra_fetch" => object(
json!({
"ra": { "type": "string", "description": "GLEIF registration-authority id to page (e.g. RA000665)." },
"out": { "type": "string", "description": "Destination CSV path, relative to the protocol directory." },
"page_size": { "type": "integer", "minimum": 1, "default": 200, "description": "GLEIF page[size]." },
"user_agent": { "type": "string", "default": "Meridian Protocol (open-analytics; research@meridian.online)", "description": "User-Agent request header." }
}),
&["ra", "out"],
),
"umap_project" => object(
json!({
"input": { "type": "string", "description": "Parquet holding the columns to project." },
"columns": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "The numeric columns to reduce, in order. A numeric scalar contributes one feature; a list/array of numerics — a vector column — contributes one per element. There is no text column and no model: to map text, write a vector column from it first (text_embed, or a SQL step calling the DuckDB embedding extension)." },
"out": { "type": "string", "description": "Parquet to write — every input column plus projection_x and projection_y as DOUBLE, and projection_fit_id (VARCHAR, same value on every row) fingerprinting the exact numbers and knobs this fit consumed." },
"fit": { "type": "string", "description": "Where the fitted projection is kept between runs, and the field that makes the map hold still when rows are appended. Declared as an asset this step both READS and PRODUCES: the first run writes it, every later run reads it back, and rows the fit already holds come back with the same coordinates while appended rows are placed into that layout instead of starting a new one. Omit it and every run refits the whole input, so appending one row moves every point. Two consequences worth knowing before you set it: deleting the file marks the step stale and refits, which is how you ask for a new layout on purpose; and moving neighbors, min_dist or metric while a fit exists is REFUSED naming the mismatch, because those knobs are part of what the fit is." },
"neighbors": { "type": "integer", "minimum": 2, "default": 15, "x-at-most": { "rows_of": "input", "minus": 1, "bounds_default": true }, "description": "UMAP n_neighbors: low reads local structure, high reads global. At most one below the input's row count: arc run counts the rows of the Parquet `input` names when the step runs, and refuses a value above that bound before the script starts, naming the value and the row count. An unset value takes 15 or that bound, whichever is smaller. x-at-most states the bound for a program: rows_of names the with: key whose Parquet is counted, minus what is taken off its row count, and bounds_default that the bound applies to the default as well. The bound is arcform's, not umap-learn's, so umap-learn's documentation for n_neighbors does not describe it." },
"min_dist": { "type": "number", "minimum": 0, "exclusiveMaximum": 1, "default": 0.1, "description": "UMAP min_dist: how tightly points may pack." },
"metric": { "type": "string", "enum": UMAP_METRICS, "default": "euclidean", "description": "How the distance between two rows is measured. The default is the reading an arbitrary feature matrix wants; cosine is the reading an L2-normalised vector column wants. Nothing here scales your columns — under euclidean a wider-spread column dominates the layout, which is a decision for the SQL step that selects them." }
}),
&["input", "columns", "out"],
),
"text_embed" => object(
json!({
"input": { "type": "string", "description": "Parquet holding the corpus." },
"text_column": { "type": "string", "description": "Column to embed." },
"extension": { "type": "string", "description": "The loadable embedding-extension artifact. A declared input: the Protocol puts it there, the operator never downloads or installs one." },
"out": { "type": "string", "description": "Parquet to write — every input column plus the vector column as FLOAT[]." },
"vector_column": { "type": "string", "default": "embedding", "description": "Column the vectors are written into. Name another to embed a second column of the same corpus in a later step." },
"model": { "type": "string", "description": "Optional. A model directory (tokenizer.json + model.safetensors + config.json) to CHECK the extension against — never read for weights, which live inside the extension. Set with model_release." },
"model_release": { "type": "string", "description": "Optional. `<model-id>@<revision>` naming the release the `model` directory was taken from. Both go into the content address the extension publishes, so the check cannot be made from the directory alone. Set with model." }
}),
&["input", "text_column", "extension", "out"],
),
"uv" => object(
json!({
"script": { "type": "string", "description": "The Python script to run, relative to the protocol directory. The Protocol carries it; this operator never fetches one." },
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The script's SHA-256, lowercase hex. Checked against the bytes on disk before uv is spawned, so `op: uv@1` addresses exact bytes the way a frozen-script operator does. Editing the script is therefore a manifest edit too; the refusal prints the digest it found." },
"deps": { "type": "array", "items": { "type": "string", "pattern": "^[A-Za-z0-9._-]+(\\[[A-Za-z0-9._,-]+\\])?==[A-Za-z0-9._+-]+$" }, "description": "Third-party requirements, each pinned `package==version`, passed to uv as --with. A range, a wildcard, an environment marker or a direct reference is refused at manifest load: any of them lets two Runs a month apart install different code while the manifest reads as unchanged." },
"args": { "type": "array", "items": { "type": "string" }, "description": "Arguments appended after the script path, verbatim and in order." },
"reads": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "What the step consumes. Required and non-empty — this is what puts the step in the asset graph, so a changed input marks it stale instead of being silently skipped. Names follow the same rule as a manifest `depends_on:`: a trailing separator is a directory, a glob metacharacter is a pattern." },
"produces": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "What the step writes. Required and non-empty, so a step downstream of this one can go stale in turn." }
}),
&["script", "sha256", "reads", "produces"],
),
"ducklake_publish" => object(
json!({
"file": { "type": "string", "description": "The built Parquet to publish, relative to the protocol directory. Declared as a read, so the step runs after whatever produces it and re-runs only when it was rebuilt. A byte-for-byte copy is registered, never the file itself: a build rewrites its output in place, and a registered file must never change." },
"catalog": { "type": "string", "description": "The DuckLake catalog: a local catalog file, relative to the protocol directory, or a DuckLake connection string such as `postgres:dbname=lake`. A leading `ducklake:` is accepted." },
"data_path": { "type": "string", "description": "Optional DATA_PATH for the attach. Omitted, DuckLake uses the path the catalog records, or `<catalog>.files/` for a new file catalog. Must be local in this version." },
"table": { "type": "string", "description": "`table` (schema main) or `schema.table`. Created from the file's schema if it does not exist. Each publish replaces the table's rows with the file's, as one snapshot; an unchanged file is a no-op that reports the snapshot already holding it." },
"credential": {
"type": "object",
"additionalProperties": false,
"description": "Optional. A DuckDB secret whose values come from the environment; the step refuses, naming the variables, when any is unset.",
"properties": {
"type": { "type": "string", "description": "The DuckDB secret TYPE: s3, r2, gcs, postgres, …" },
"env": { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" }, "description": "Secret parameter → the environment variable holding its value, e.g. `key_id: R2_ACCESS_KEY_ID`." }
},
"required": ["type", "env"]
}
}),
&["file", "catalog", "table"],
),
_ => return None,
};
// Name the schema after the operator so an authoring form can title it.
let mut schema = schema;
if let Some(map) = schema.as_object_mut() {
map.insert("title".to_string(), json!(op_name));
}
Some(schema)
}
/// One `with:` key an operator's step reads or writes through: the key, what kind of
/// thing its value names, the key of the read a column belongs to, and one sentence.
/// Whether it is a list, whether it is required and a list's fewest values are read
/// from the key's entry in [`with_schema`], so they are stated once.
struct RoleKey {
key: &'static str,
kind: &'static str,
of: Option<&'static str>,
description: &'static str,
}
/// What an operator's description says beyond its `with:` schema: one sentence
/// saying what it does, and the keys its step reads and writes through.
struct Described {
name: &'static str,
summary: &'static str,
reads: &'static [RoleKey],
writes: &'static [RoleKey],
}
/// A role whose value names an asset of `kind`, the way the operator's `assets()`
/// records it.
const fn role(key: &'static str, kind: &'static str, description: &'static str) -> RoleKey {
RoleKey {
key,
kind,
of: None,
description,
}
}
/// A role whose value names a column, or a list of columns, of the read `of`.
const fn column_of(key: &'static str, of: &'static str, description: &'static str) -> RoleKey {
RoleKey {
key,
kind: "column",
of: Some(of),
description,
}
}
/// The summary and the roles of each operator in the catalog. A role is a `with:` key its `assets()` records a read or a
/// produced asset from, with that asset's kind, or a key naming columns of one; a
/// path the step uses that `assets()` does not record stays in `parameters`:
/// `finetype_validate`'s `extension`, `uv`'s `script` and `ducklake_publish`'s
/// `catalog`. `archive_extract` records `dest` as the directory when `members` is
/// empty and as each file under it when `members` names them, so `dest` is one
/// `directory` role. `uv` takes each name's kind from how it is written, so its roles
/// are `any`.
const DESCRIBED: &[Described] = &[
Described {
name: "parquet_export",
summary: "Writes a table of the pipeline's database to a Parquet file.",
reads: &[role(
"input",
"table",
"The table or view in the pipeline's database to export.",
)],
writes: &[role(
"dest",
"file",
"The Parquet file written, relative to the protocol directory.",
)],
},
Described {
name: "http_fetch",
summary: "Downloads the file at one HTTP or HTTPS address.",
reads: &[],
writes: &[role(
"out",
"file",
"The file the download is written to, relative to the protocol directory.",
)],
},
Described {
name: "opendal_fetch",
summary: "Copies one object from a storage backend, such as S3 or HTTPS, to a file.",
reads: &[],
writes: &[role(
"to",
"file",
"The file the object is written to, relative to the protocol directory.",
)],
},
Described {
name: "html_link_discover",
summary: "Fetches an index page and lists the links on it that match a pattern, one address per line.",
reads: &[],
writes: &[role(
"out",
"file",
"The file the matching addresses are written to, one per line.",
)],
},
Described {
name: "archive_extract",
summary: "Extracts members of a zip archive into a directory.",
reads: &[role(
"archive",
"file",
"The zip archive to extract, relative to the protocol directory.",
)],
writes: &[role(
"dest",
"directory",
"The directory the members are extracted into, recorded as each file under it when `members` names them.",
)],
},
Described {
name: "datapackage_describe",
summary: "Writes a Data Package descriptor for a Parquet file, its columns typed by FineType and overlaid with curated overrides.",
reads: &[
role(
"parquet",
"file",
"The Parquet file whose columns FineType types.",
),
role(
"overrides",
"file",
"The curated descriptor sidecar, in JSON, overlaid onto FineType's descriptor.",
),
role(
"nominations",
"file",
"The declared column types FineType takes as given instead of inferring them.",
),
],
writes: &[role(
"out",
"file",
"The Data Package descriptor written, in JSON.",
)],
},
Described {
name: "finetype_validate",
summary: "Checks a Parquet file against a JSON Schema contract with FineType, failing the step on any rejected row.",
reads: &[
role("parquet", "file", "The Parquet file to check."),
role(
"schema",
"file",
"The JSON Schema contract the file is checked against.",
),
],
writes: &[],
},
Described {
name: "splink_resolve",
summary: "Matches EDGAR entities to GLEIF entities with Splink and writes the resolved crosswalk.",
reads: &[
role(
"edgar",
"file",
"The EDGAR entity Parquet file, the crosswalk's left side.",
),
role(
"gleif",
"file",
"The GLEIF golden-copy Parquet file, the crosswalk's right side.",
),
],
writes: &[role(
"out",
"file",
"The Parquet file the resolved crosswalk is written to.",
)],
},
Described {
name: "gleif_ra_fetch",
summary: "Pages through GLEIF's entities for one registration authority and writes them to a CSV file.",
reads: &[],
writes: &[role(
"out",
"file",
"The CSV file the entities are written to, relative to the protocol directory.",
)],
},
Described {
name: "umap_project",
summary: "Places each row of a Parquet file on a two-dimensional map with UMAP, from the numeric columns it names.",
reads: &[
role(
"input",
"file",
"The Parquet file holding the columns to project.",
),
column_of(
"columns",
"input",
"The numeric columns of `input` to reduce, in order.",
),
role(
"fit",
"file",
"The fit an earlier run kept, read back so the rows it holds keep their places.",
),
],
writes: &[
role(
"out",
"file",
"The Parquet file written, every input column plus each row's place on the map.",
),
role(
"fit",
"file",
"The fit this run keeps, so a later run places appended rows into the same layout.",
),
],
},
Described {
name: "text_embed",
summary: "Writes a vector for the text in one column of a Parquet file, with a DuckDB embedding extension.",
reads: &[
role("input", "file", "The Parquet file holding the corpus."),
column_of(
"text_column",
"input",
"The column of `input` whose text is embedded.",
),
role(
"extension",
"file",
"The loadable embedding extension, which holds the model's weights.",
),
role(
"model",
"directory",
"A model directory the extension is checked against, never read for weights.",
),
],
writes: &[role(
"out",
"file",
"The Parquet file written, every input column plus the vector column.",
)],
},
Described {
name: "uv",
summary: "Runs a Python script pinned by its digest with uv.",
reads: &[role(
"reads",
"any",
"What the script consumes, each name's kind read from how it is written.",
)],
writes: &[role(
"produces",
"any",
"What the script writes, each name's kind read from how it is written.",
)],
},
Described {
name: "ducklake_publish",
summary: "Publishes a Parquet file as the rows of a table in a DuckLake catalog, as one snapshot.",
reads: &[role(
"file",
"file",
"The built Parquet file to publish, of which a byte-for-byte copy is registered.",
)],
writes: &[],
},
];
/// The summary and the roles of the operator called `op_name`, or `None` for a name
/// [`DESCRIBED`] does not hold.
fn described(op_name: &str) -> Option<&'static Described> {
DESCRIBED.iter().find(|d| d.name == op_name)
}
/// The description `arc operation describe` prints for the operator called
/// `op_name`, in the shape it prints for an operation, or `None` when this build's
/// catalog does not hold it.
///
/// Its roles are the `with:` keys [`described`] names; `parameters` is the operator's
/// [`with_schema`] with those keys taken out of `properties` and `required` and
/// nothing else changed. `arc mcp`'s `operator_describe` returns the schema whole,
/// the form a `with:` block is written from.
pub(crate) fn description(op_name: &str) -> Option<serde_json::Value> {
catalog().into_iter().find(|op| op.name() == op_name)?;
// Both tables answer for every catalog operator, as the tests below hold them to.
let (described, schema) = described(op_name).zip(with_schema(op_name))?;
let roles = |keys: &[RoleKey]| -> Vec<crate::record::Role> {
keys.iter().map(|k| resolved_role(k, &schema)).collect()
};
let reads = roles(described.reads);
let writes = roles(described.writes);
let mut parameters = schema.clone();
let is_role = |key: &str| {
described
.reads
.iter()
.chain(described.writes)
.any(|r| r.key == key)
};
if let Some(properties) = parameters["properties"].as_object_mut() {
properties.retain(|key, _| !is_role(key));
}
if let Some(required) = parameters["required"].as_array_mut() {
required.retain(|key| !key.as_str().is_some_and(is_role));
}
Some(crate::record::description(
op_name,
described.summary,
&reads,
&writes,
parameters,
))
}
/// `key` as a role, its `list`, `required` and `min_items` read from its entry in
/// the operator's `with:` schema.
fn resolved_role(key: &RoleKey, schema: &serde_json::Value) -> crate::record::Role {
let entry = &schema["properties"][key.key];
let list = entry["type"] == "array";
crate::record::Role {
name: key.key,
kind: key.kind,
list,
required: schema["required"]
.as_array()
.is_some_and(|required| required.iter().any(|k| k == key.key)),
of: key.of,
min_items: list.then(|| entry["minItems"].as_u64().unwrap_or(0)),
description: key.description,
}
}
#[cfg(test)]
mod with_schema_tests {
use super::*;
#[test]
fn every_catalog_operator_has_a_with_schema() {
for name in catalog_names() {
let schema = with_schema(name)
.unwrap_or_else(|| panic!("operator `{name}` is in the catalog but has no `with:` schema — add one in `with_schema`"));
assert_eq!(
schema["type"], "object",
"`{name}` schema must be an object"
);
assert!(
schema["properties"].is_object(),
"`{name}` schema must carry `properties`"
);
}
}
#[test]
fn unknown_operator_has_no_schema() {
assert!(with_schema("does_not_exist").is_none());
}
#[test]
fn ducklake_publish_schema_shape() {
let schema = with_schema("ducklake_publish").expect("ducklake_publish has a schema");
let required: Vec<&str> = schema["required"]
.as_array()
.unwrap()
.iter()
.filter_map(|v| v.as_str())
.collect();
assert_eq!(required, ["file", "catalog", "table"]);
// The credential is typed config like the rest: an authoring form offering a
// key the operator's `deny_unknown_fields` would refuse builds a `with:` block
// that will not load.
let credential = &schema["properties"]["credential"];
assert_eq!(credential["additionalProperties"], false);
assert_eq!(credential["required"], serde_json::json!(["type", "env"]));
assert_eq!(
credential["properties"]["env"]["additionalProperties"]["type"],
"string"
);
}
#[test]
fn parquet_export_schema_shape() {
let schema = with_schema("parquet_export").expect("parquet_export has a schema");
assert_eq!(schema["type"], "object");
assert!(schema["properties"]["input"].is_object());
assert!(schema["properties"]["dest"].is_object());
let required: Vec<&str> = schema["required"]
.as_array()
.unwrap()
.iter()
.filter_map(|v| v.as_str())
.collect();
assert!(required.contains(&"input") && required.contains(&"dest"));
// `metadata` is optional, and an authoring form has to know it is a
// string→string mapping: the config is `BTreeMap<String, String>`, so a
// form offering any other value type builds a `with:` block that will not
// deserialize.
let metadata = &schema["properties"]["metadata"];
assert_eq!(metadata["type"], "object", "`metadata` must be an object");
assert_eq!(
metadata["additionalProperties"]["type"], "string",
"`metadata` values must be declared as strings"
);
assert!(
!required.contains(&"metadata"),
"`metadata` must stay optional — an export declaring none must remain valid"
);
}
// ── Each schema holds to what its operator reads ──────────────────────────────
//
// `with_schema` is written by hand, and the typed deserialize stays the gate, so
// nothing but these tests keeps the two in step. Each one reads the configuration
// by running it: the fields a derive asks serde for, and whether `assets_for` — the
// `parse`, then `validate()`, then pattern compile that manifest load runs — admits
// a manifest built from the schema's own values. The four frozen-script operators
// are held to the script as well, read out of the file their argv names.
use std::cell::RefCell;
use std::collections::BTreeSet;
use serde::de::{DeserializeSeed, MapAccess, Visitor};
use serde_json::{Value as Json, json};
/// The fields a configuration's derive asks serde for, at each struct it
/// deserializes, by path: `""` for the `with:` block, `"credential"` for a struct
/// inside it.
///
/// Read from the derive rather than the source: a derived `Deserialize` hands
/// `deserialize_struct` the names it accepts, renames applied, which are the keys
/// `deny_unknown_fields` admits. The recorder answers every field with a
/// placeholder of the type asked for, so the derive carries on into a nested struct
/// and names its fields too. A struct inside a sequence or a map is not reached,
/// because no element is produced; no configuration has one, and [`schema_objects`]
/// refuses a schema that would need it.
fn configuration_fields<T: serde::de::DeserializeOwned>() -> BTreeMap<String, BTreeSet<String>>
{
let found = RefCell::new(BTreeMap::new());
let recorder = FieldRecorder {
path: String::new(),
found: &found,
};
if let Err(e) = T::deserialize(recorder) {
panic!(
"recording the fields of {}: {e}",
std::any::type_name::<T>()
);
}
found.into_inner()
}
/// Each catalogue operator's configuration struct, recorded. An operator missing
/// here fails the test that asks for it, so a new one cannot pass unread.
fn configuration_fields_of(op: &str) -> BTreeMap<String, BTreeSet<String>> {
match op {
"parquet_export" => configuration_fields::<ParquetExportConfig>(),
#[cfg(feature = "http-fetch")]
"http_fetch" => configuration_fields::<HttpFetchConfig>(),
#[cfg(feature = "opendal")]
"opendal_fetch" => configuration_fields::<OpendalFetchConfig>(),
#[cfg(feature = "http-fetch")]
"html_link_discover" => configuration_fields::<HtmlLinkDiscoverConfig>(),
"archive_extract" => configuration_fields::<ArchiveExtractConfig>(),
"datapackage_describe" => configuration_fields::<DatapackageDescribeConfig>(),
"finetype_validate" => configuration_fields::<FinetypeValidateConfig>(),
"splink_resolve" => configuration_fields::<SplinkResolveConfig>(),
"gleif_ra_fetch" => configuration_fields::<GleifRaFetchConfig>(),
"umap_project" => configuration_fields::<UmapProjectConfig>(),
"text_embed" => configuration_fields::<TextEmbedConfig>(),
"uv" => configuration_fields::<UvConfig>(),
"ducklake_publish" => configuration_fields::<ducklake_publish::DucklakePublishConfig>(),
other => panic!(
"operator `{other}` is in the catalogue, but configuration_fields_of does \
not name its configuration struct — add it there"
),
}
}
struct FieldRecorder<'a> {
path: String,
found: &'a RefCell<BTreeMap<String, BTreeSet<String>>>,
}
macro_rules! placeholder {
($($method:ident => $visit:ident($($value:expr)?)),* $(,)?) => {$(
fn $method<V: Visitor<'de>>(self, visitor: V) -> std::result::Result<V::Value, Self::Error> {
visitor.$visit($($value)?)
}
)*};
}
impl<'de> serde::Deserializer<'de> for FieldRecorder<'_> {
type Error = serde::de::value::Error;
fn deserialize_any<V: Visitor<'de>>(
self,
_: V,
) -> std::result::Result<V::Value, Self::Error> {
Err(serde::de::Error::custom(format!(
"`{}` has a type the field recorder has no placeholder for",
self.path
)))
}
placeholder! {
deserialize_bool => visit_bool(false),
deserialize_u8 => visit_u64(0),
deserialize_u16 => visit_u64(0),
deserialize_u32 => visit_u64(0),
deserialize_u64 => visit_u64(0),
deserialize_i8 => visit_u64(0),
deserialize_i16 => visit_u64(0),
deserialize_i32 => visit_u64(0),
deserialize_i64 => visit_u64(0),
deserialize_f32 => visit_f64(0.0),
deserialize_f64 => visit_f64(0.0),
deserialize_str => visit_str(""),
deserialize_string => visit_str(""),
deserialize_unit => visit_unit(),
deserialize_seq => visit_seq(serde::de::value::SeqDeserializer::new(std::iter::empty::<u8>())),
deserialize_map => visit_map(serde::de::value::MapDeserializer::new(std::iter::empty::<(u8, u8)>())),
}
fn deserialize_option<V: Visitor<'de>>(
self,
visitor: V,
) -> std::result::Result<V::Value, Self::Error> {
visitor.visit_some(self)
}
fn deserialize_newtype_struct<V: Visitor<'de>>(
self,
_name: &'static str,
visitor: V,
) -> std::result::Result<V::Value, Self::Error> {
visitor.visit_newtype_struct(self)
}
fn deserialize_struct<V: Visitor<'de>>(
self,
_name: &'static str,
fields: &'static [&'static str],
visitor: V,
) -> std::result::Result<V::Value, Self::Error> {
self.found.borrow_mut().insert(
self.path.clone(),
fields.iter().map(|f| f.to_string()).collect(),
);
visitor.visit_map(StructFields {
path: self.path,
fields: fields.iter(),
current: "",
found: self.found,
})
}
serde::forward_to_deserialize_any! {
i128 u128 char bytes byte_buf unit_struct tuple tuple_struct enum identifier ignored_any
}
}
struct StructFields<'a> {
path: String,
fields: std::slice::Iter<'static, &'static str>,
current: &'static str,
found: &'a RefCell<BTreeMap<String, BTreeSet<String>>>,
}
impl<'de> MapAccess<'de> for StructFields<'_> {
type Error = serde::de::value::Error;
fn next_key_seed<K: DeserializeSeed<'de>>(
&mut self,
seed: K,
) -> std::result::Result<Option<K::Value>, Self::Error> {
let Some(field) = self.fields.next() else {
return Ok(None);
};
self.current = field;
seed.deserialize(serde::de::value::BorrowedStrDeserializer::new(field))
.map(Some)
}
fn next_value_seed<V: DeserializeSeed<'de>>(
&mut self,
seed: V,
) -> std::result::Result<V::Value, Self::Error> {
seed.deserialize(FieldRecorder {
path: joined(&self.path, self.current),
found: self.found,
})
}
}
fn joined(path: &str, key: &str) -> String {
if path.is_empty() {
key.to_string()
} else {
format!("{path}.{key}")
}
}
/// Each object a `with:` schema describes, by the path its keys are read at: `""`
/// for the block, and the path of every property that is an object with
/// `properties` of its own.
fn schema_objects(schema: &Json) -> BTreeMap<String, &Json> {
fn walk<'s>(path: String, object: &'s Json, out: &mut BTreeMap<String, &'s Json>) {
for (key, prop) in properties(object) {
for nested in ["items", "additionalProperties"] {
assert!(
prop[nested].get("properties").is_none(),
"`{}` describes an object inside its `{nested}`, which the \
field recorder does not reach — extend it before relying on \
this check there",
joined(&path, key)
);
}
if prop.get("properties").is_some() {
walk(joined(&path, key), prop, out);
}
}
out.insert(path, object);
}
let mut out = BTreeMap::new();
walk(String::new(), schema, &mut out);
out
}
fn properties(object: &Json) -> Vec<(&String, &Json)> {
object["properties"]
.as_object()
.map(|p| p.iter().collect())
.unwrap_or_default()
}
fn required(object: &Json) -> BTreeSet<String> {
object["required"]
.as_array()
.map(|r| {
r.iter()
.filter_map(|k| k.as_str().map(String::from))
.collect()
})
.unwrap_or_default()
}
/// A value `prop` admits, built from the schema alone: the last `enum` entry, a
/// number inside its bounds, two items for an array, every property for an object.
/// Each string carries `key`, so the script check can find in the argv where a
/// key's value went, and is an identifier, which the configurations that check a
/// name (`credential.type`, `table`) admit. A string with a `pattern` takes the
/// first candidate the pattern matches.
fn probe(key: &str, prop: &Json) -> Json {
if let Some(values) = prop["enum"].as_array() {
return values
.last()
.cloned()
.expect("an empty enum admits nothing");
}
match prop["type"].as_str() {
Some("string") => {
let candidates = [
format!("probe_{key}"),
"a".repeat(64),
format!("probe_{key}==1.0"),
];
let Some(pattern) = prop["pattern"].as_str() else {
return json!(candidates[0]);
};
let pattern = regex::Regex::new(pattern).expect("a schema pattern compiles");
let hit = candidates.iter().find(|c| pattern.is_match(c));
json!(hit.unwrap_or_else(|| panic!(
"no probe value for `{key}` matches its pattern {pattern} — add a candidate to probe()"
)))
}
Some("integer") => json!(prop["minimum"].as_i64().map_or(1, |m| m + 1)),
Some("number") => {
let low = prop["minimum"].as_f64().unwrap_or(0.0);
let high = prop["exclusiveMaximum"]
.as_f64()
.or(prop["maximum"].as_f64())
.unwrap_or(low + 1.0);
json!((low + high) / 2.0)
}
Some("boolean") => json!(true),
Some("array") => json!([
probe(&format!("{key}_0"), &prop["items"]),
probe(&format!("{key}_1"), &prop["items"]),
]),
Some("object") => match prop.get("properties") {
Some(_) => properties(prop)
.into_iter()
.map(|(k, p)| (k.clone(), probe(k, p)))
.collect::<serde_json::Map<_, _>>()
.into(),
None => json!({
format!("probe_{key}"): probe(&format!("{key}_value"), &prop["additionalProperties"])
}),
},
other => panic!("`{key}` has schema type {other:?}, which probe() cannot build"),
}
}
/// `full` with the object at `path` replaced by `object`.
fn with_object_at(full: &Json, path: &str, object: Json) -> Json {
if path.is_empty() {
return object;
}
let mut out = full.clone();
let pointer = format!("/{}", path.replace('.', "/"));
*out.pointer_mut(&pointer)
.expect("the path is in the manifest") = object;
out
}
/// `full` with the key `key` of the object at `path` set to `value`.
fn with_key_at(full: &Json, path: &str, key: &str, value: Json) -> Json {
let mut object = if path.is_empty() {
full.clone()
} else {
full.pointer(&format!("/{}", path.replace('.', "/")))
.expect("the path is in the manifest")
.clone()
};
object[key] = value;
with_object_at(full, path, object)
}
/// Whether manifest load admits `with` for `op`: the typed deserialize, then
/// `validate()` where there is one, then the pattern compile, as `assets_for` runs
/// them.
fn admits(op: &str, with: &Json) -> std::result::Result<(), String> {
let with: Value = serde_yaml::to_value(with).expect("a JSON value converts to YAML");
assets_for(op, Some(&with))
.map(|_| ())
.map_err(|e| e.to_string())
}
fn fail_on(faults: &[String], what: &str) {
assert!(faults.is_empty(), "{what}:\n {}", faults.join("\n "));
}
/// A key the schema names is one the configuration reads, and the reverse, at
/// every object the schema describes; and a schema closed with
/// `additionalProperties: false` is closed in the configuration too.
#[test]
fn every_operators_schema_names_the_keys_its_configuration_reads() {
let mut faults = Vec::new();
for op in catalog_names() {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let objects = schema_objects(&schema);
let fields = configuration_fields_of(op);
let paths: BTreeSet<&String> = objects.keys().chain(fields.keys()).collect();
for path in paths {
let named: BTreeSet<String> = objects
.get(path)
.map(|o| properties(o).into_iter().map(|(k, _)| k.clone()).collect())
.unwrap_or_default();
let read = fields.get(path).cloned().unwrap_or_default();
for key in named.difference(&read) {
faults.push(format!(
"{op}: the schema names `{}`, which the configuration refuses as an unknown key",
joined(path, key)
));
}
for key in read.difference(&named) {
faults.push(format!(
"{op}: the configuration reads `{}`, which the schema does not name",
joined(path, key)
));
}
}
let full = probe("", &schema);
if admits(op, &full).is_err() {
// Every probe would be refused whatever it held, so there is nothing to
// read closure from; the type test says why the manifest is refused.
continue;
}
for (path, object) in &objects {
let closed = object["additionalProperties"] == json!(false);
let stray = with_key_at(&full, path, "probe_unknown_key", json!("probe"));
let refused = admits(op, &stray).is_err();
if closed != refused {
faults.push(format!(
"{op}: the schema's object at `{path}` is {}, but the configuration {} a key it does not read",
if closed { "closed" } else { "open" },
if refused { "refuses" } else { "admits" },
));
}
}
}
fail_on(
&faults,
"a schema names a key its configuration does not read, or the reverse",
);
}
/// A key is in the schema's `required` exactly when every manifest the
/// configuration admits holds it. Read by building every subset of an object's
/// keys at their probe values and asking manifest load which it admits: removing
/// one key from a full manifest is not the same question, since `text_embed`
/// refuses `model` without `model_release`, and neither is the required keys
/// alone, since `archive_extract` wants `members` or `pattern`.
#[test]
fn every_operators_schema_requires_the_keys_every_admitted_manifest_holds() {
let mut faults = Vec::new();
for op in catalog_names() {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let full = probe("", &schema);
for (path, object) in schema_objects(&schema) {
let keys = properties(object);
assert!(
keys.len() < 16,
"{op}: `{path}` has too many keys to enumerate"
);
let declared = required(object);
for key in &declared {
if !keys.iter().any(|(k, _)| *k == key) {
faults.push(format!(
"{op}: the schema requires `{}`, which it does not describe",
joined(&path, key)
));
}
}
let subset = |mask: u32| -> Json {
keys.iter()
.enumerate()
.filter(|(i, _)| mask & (1 << i) != 0)
.map(|(_, (k, p))| ((*k).clone(), probe(k, p)))
.collect::<serde_json::Map<_, _>>()
.into()
};
let admitted: Vec<u32> = (0..1u32 << keys.len())
.filter(|&mask| admits(op, &with_object_at(&full, &path, subset(mask))).is_ok())
.collect();
if admitted.is_empty() {
faults.push(format!(
"{op}: no manifest built from the schema's keys at `{path}` is admitted, so \
which keys are required cannot be read; the full manifest is refused: {}",
admits(op, &full).err().unwrap_or_default()
));
continue;
}
for (i, (key, _)) in keys.iter().enumerate() {
let omitted_by = admitted.iter().find(|&&mask| mask & (1 << i) == 0);
match (omitted_by, declared.contains(*key)) {
(None, false) => faults.push(format!(
"{op}: every manifest the configuration admits holds `{}`, but the \
schema does not list it in `required`",
joined(&path, key)
)),
(Some(&mask), true) => faults.push(format!(
"{op}: the schema requires `{}`, but the configuration admits a manifest \
without it: {}",
joined(&path, key),
subset(mask)
)),
_ => {}
}
}
}
}
fail_on(
&faults,
"a schema's `required` disagrees with what its configuration admits",
);
}
// ── Each description holds to its schema and to what its step records ─────────
//
// An operator's description names the `with:` keys its step reads and writes
// through, and takes them out of its `with:` schema to leave `parameters`. The
// roles are written by hand beside `with_schema`, so these tests read them against
// the two things that know: the schema, and `assets()` run on manifests the
// configuration admits.
/// The word a description uses for an asset of `kind`.
fn kind_word(kind: crate::asset_kind::AssetKind) -> &'static str {
use crate::asset_kind::AssetKind;
match kind {
AssetKind::File => "file",
AssetKind::Directory => "directory",
AssetKind::Pattern => "pattern",
AssetKind::Table => "table",
}
}
/// The names of the roles in `description[direction]`, with each role's kind.
fn roles_of<'d>(description: &'d Json, direction: &str) -> Vec<(&'d str, &'d str)> {
description[direction]
.as_array()
.unwrap_or_else(|| panic!("`{direction}` is not a list: {description}"))
.iter()
.map(|r| {
(
r["name"].as_str().expect("a role has a name"),
r["kind"].as_str().expect("a role has a kind"),
)
})
.collect()
}
#[test]
fn every_catalog_operator_is_described_and_every_description_names_one() {
for op in catalog_names() {
assert!(
description(op).is_some(),
"operator `{op}` is in the catalogue but has no description — add it to DESCRIBED"
);
}
for described in DESCRIBED {
assert!(
with_schema(described.name).is_some(),
"DESCRIBED holds `{}`, which is no operator",
described.name
);
}
assert!(
description("does_not_exist").is_none(),
"a name no operator has is described"
);
}
/// `parameters` is the `with:` schema with the role keys taken out of
/// `properties` and `required`, and nothing else changed; each role is a key the
/// schema describes, and its `list`, `required` and `min_items` are what the
/// schema says of that key.
#[test]
fn every_operators_parameters_are_its_with_schema_less_its_roles() {
let mut faults = Vec::new();
for op in catalog_names() {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let described = description(op).expect("every catalogue operator is described");
let parameters = &described["parameters"];
let roles: BTreeSet<&str> = roles_of(&described, "reads")
.into_iter()
.chain(roles_of(&described, "writes"))
.map(|(name, _)| name)
.collect();
let declared = required(&schema);
for direction in ["reads", "writes"] {
for role in described[direction].as_array().expect("a list") {
let name = role["name"].as_str().expect("a name");
let Some(entry) = schema["properties"].get(name) else {
faults.push(format!(
"{op}: {direction} `{name}`, which its `with:` schema does not describe"
));
continue;
};
let list = entry["type"] == "array";
let expected = (
list,
declared.contains(name),
list.then(|| entry["minItems"].as_u64().unwrap_or(0)),
);
let stated = (
role["list"].as_bool().unwrap_or(!list),
role["required"].as_bool().unwrap_or(!expected.1),
role["min_items"].as_u64(),
);
if stated != expected {
faults.push(format!(
"{op}: {direction} `{name}` states (list, required, min_items) \
{stated:?}, and its `with:` schema says {expected:?}"
));
}
}
}
let mut rebuilt = schema.clone();
rebuilt["properties"]
.as_object_mut()
.expect("a schema has properties")
.retain(|key, _| !roles.contains(key.as_str()));
rebuilt["required"]
.as_array_mut()
.expect("a schema has required")
.retain(|key| !roles.contains(key.as_str().unwrap_or_default()));
if *parameters != rebuilt {
faults.push(format!(
"{op}: `parameters` is not its `with:` schema less the roles {roles:?}:\n \
parameters: {parameters}\n expected: {rebuilt}"
));
}
}
fail_on(
&faults,
"an operator's `parameters` or roles disagree with its `with:` schema",
);
}
#[test]
fn umap_projects_parameters_keep_neighbors_whole_and_its_bound_names_a_read() {
let described = description("umap_project").expect("umap_project is described");
let properties = described["parameters"]["properties"]
.as_object()
.expect("`parameters` has properties");
for role in ["input", "columns", "out", "fit"] {
assert!(
!properties.contains_key(role),
"`{role}` is a role and stays in `parameters`: {properties:?}"
);
}
let schema = with_schema("umap_project").expect("umap_project has a schema");
assert_eq!(
properties["neighbors"], schema["properties"]["neighbors"],
"`neighbors` must reach `parameters` whole, its `default` and `x-at-most` with it"
);
let rows_of = properties["neighbors"]["x-at-most"]["rows_of"]
.as_str()
.expect("`x-at-most` names the key whose rows bound it");
assert!(
roles_of(&described, "reads").contains(&(rows_of, "file")),
"`x-at-most` counts the rows of `{rows_of}`, which is no file the step reads"
);
let described = description("splink_resolve").expect("splink_resolve is described");
for role in ["edgar", "gleif", "out"] {
assert!(
described["parameters"]["properties"].get(role).is_none()
&& !described["parameters"]["required"]
.as_array()
.expect("`required` is a list")
.contains(&json!(role)),
"`{role}` is a role and stays in splink_resolve's `parameters`: {described}"
);
}
}
/// Every name `assets()` records comes from a key the description lists in the
/// same direction, with the kind the role states or any kind for `any`; and every
/// role other than a column is one `assets()` records from when it is set. Read on
/// every manifest built from the schema's keys at their probe values that manifest
/// load admits, so each optional role is read set and unset, and `archive_extract`
/// with `members` and with `pattern`. A file recorded under the directory a
/// `directory` role names counts as that role: `archive_extract`'s `dest` with
/// `members` set.
#[test]
fn every_operators_roles_are_the_keys_its_assets_record_from() {
let mut faults = Vec::new();
for op in catalog_names() {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let described = description(op).expect("every catalogue operator is described");
let keys = properties(&schema);
let role_keys: BTreeSet<&str> = roles_of(&described, "reads")
.into_iter()
.chain(roles_of(&described, "writes"))
.map(|(name, _)| name)
.collect();
let manifest = |mask: u32| -> Json {
keys.iter()
.enumerate()
.filter(|(i, _)| mask & (1 << i) != 0)
.map(|(_, (k, p))| ((*k).clone(), probe(k, p)))
.collect::<serde_json::Map<_, _>>()
.into()
};
let mut every_role_set = false;
for mask in 0..1u32 << keys.len() {
let with = manifest(mask);
let yaml: Value = serde_yaml::to_value(&with).expect("JSON converts to YAML");
let Ok(assets) = resolve(op).expect("in the catalogue").assets(&yaml) else {
continue;
};
every_role_set |= role_keys.iter().all(|k| with.get(*k).is_some());
for (direction, recorded) in
[("reads", &assets.reads), ("writes", &assets.produces)]
{
let roles = roles_of(&described, direction);
let from = |name: &str, role: &str, role_kind: &str| -> bool {
let kind = kind_word(assets.kinds[name]);
let kind_holds = role_kind == "any" || role_kind == kind;
match &with[role] {
Json::String(value) => {
(name == value && kind_holds)
|| (role_kind == "directory"
&& kind == "file"
&& name.starts_with(&format!(
"{}/",
value.trim_end_matches('/')
)))
}
Json::Array(values) => kind_holds && values.iter().any(|v| v == name),
_ => false,
}
};
for name in recorded {
if !roles.iter().any(|(role, kind)| from(name, role, kind)) {
faults.push(format!(
"{op}: `assets()` records `{name}` ({}) as {direction}, from no \
key the description lists in `{direction}` with that kind; \
manifest {with}",
kind_word(assets.kinds[name])
));
}
}
for (role, kind) in &roles {
if *kind == "column" || with.get(*role).is_none() {
continue;
}
if !recorded.iter().any(|name| from(name, role, kind)) {
faults.push(format!(
"{op}: `{role}` is listed in `{direction}`, and `assets()` \
records nothing from it as {direction}; manifest {with}"
));
}
}
}
}
if !every_role_set {
faults.push(format!(
"{op}: no manifest the configuration admits sets every role {role_keys:?}, \
so an optional role was never read against `assets()`"
));
}
}
fail_on(
&faults,
"an operator's roles disagree with what its `assets()` records",
);
}
const JSON_TYPES: [&str; 6] = ["string", "integer", "number", "boolean", "array", "object"];
fn generic_value(json_type: &str) -> Json {
match json_type {
"string" => json!("probe"),
"integer" => json!(3),
"number" => json!(0.5),
"boolean" => json!(true),
"array" => json!(["probe"]),
_ => json!({ "probe": "probe" }),
}
}
/// Whether a schema typed `declared` admits a value of `json_type`: an integer is a
/// number too.
fn schema_admits(declared: &str, json_type: &str) -> bool {
declared == json_type || (declared == "number" && json_type == "integer")
}
/// A key's schema type is the type its configuration admits it as. A manifest of
/// every key at a value of its schema type is admitted; when it is not, the key at
/// fault is found by giving it a value of each other type in turn. And a value of a
/// type the schema does not admit is refused, inside a manifest that is otherwise
/// admitted. A `null` is not probed: YAML writes an unset optional value that way,
/// and serde reads it as unset.
#[test]
fn every_operators_schema_types_agree_with_what_its_configuration_admits() {
let mut faults = Vec::new();
for op in catalog_names() {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let full = probe("", &schema);
let full_admitted = admits(op, &full);
if let Err(e) = &full_admitted {
faults.push(format!(
"{op}: a manifest holding every key the schema names, each at a value of its \
schema type, is refused: {e}"
));
}
for (path, object) in schema_objects(&schema) {
for (key, prop) in properties(object) {
let declared = prop["type"].as_str().unwrap_or_else(|| {
panic!("{op}: `{}` has no schema type", joined(&path, key))
});
let at = |value: Json| with_key_at(&full, &path, key, value);
if full_admitted.is_ok() {
for json_type in JSON_TYPES.iter().filter(|t| !schema_admits(declared, t)) {
if admits(op, &at(generic_value(json_type))).is_ok() {
faults.push(format!(
"{op}: the schema types `{}` as {declared}, but the configuration \
also admits it as {json_type} ({})",
joined(&path, key),
generic_value(json_type)
));
}
}
} else {
let admitted_as: Vec<&str> = JSON_TYPES
.into_iter()
.filter(|t| admits(op, &at(generic_value(t))).is_ok())
.collect();
if !admitted_as.is_empty() {
faults.push(format!(
"{op}: the configuration refuses `{}` as the schema's {declared} ({}) \
and admits it as {}",
joined(&path, key),
probe(key, prop),
admitted_as.join(" or ")
));
}
}
}
}
}
fail_on(
&faults,
"a schema's type for a key disagrees with what its configuration admits",
);
}
/// The operators that run a frozen Python script.
const SCRIPT_OPERATORS: [&str; 4] = [
"splink_resolve",
"gleif_ra_fetch",
"umap_project",
"text_embed",
];
/// Options a frozen script declares that no `with:` key reaches, each with the
/// reason arc never passes it. Every run through arc takes the script's default for
/// these, so none is a value an analyst could set and not see.
const OPTIONS_ARC_NEVER_PASSES: &[(&str, &str, &str)] = &[(
"gleif_ra_fetch",
"--max-pages",
"a smoke-test lever only: production always does a full, guarded, unbounded \
pull, so every run through arc takes the script's default of no cap",
)];
/// The argv a frozen-script operator's `run` hands `uv` for `with`, built by the
/// function `run` calls, without spawning `uv`.
fn script_invocation(op: &str, with: &Value, dir: &Path) -> Result<Vec<String>> {
match op {
"splink_resolve" => splink_resolve_invocation(&SplinkResolveConfig::parse(with)?, dir),
"gleif_ra_fetch" => gleif_ra_fetch_invocation(&GleifRaFetchConfig::parse(with)?, dir),
"umap_project" => {
let cfg = UmapProjectConfig::parse(with)?;
cfg.validate()?;
umap_project_invocation(&cfg, dir)
}
"text_embed" => {
let cfg = TextEmbedConfig::parse(with)?;
cfg.validate()?;
// The extension and the model are declared inputs the invocation
// refuses to go without, so put them where the manifest says.
std::fs::write(dir.join(&cfg.extension), b"").unwrap();
if let Some(model) = &cfg.model {
let model = dir.join(model);
std::fs::create_dir_all(&model).unwrap();
for part in MODEL_PARTS {
std::fs::write(model.join(part), b"").unwrap();
}
}
text_embed_invocation(&cfg, dir)
}
other => panic!("`{other}` is not a frozen-script operator"),
}
}
/// One `add_argument(...)` call in a Python source.
#[derive(Debug)]
struct ScriptOption {
flags: Vec<String>,
required: bool,
type_name: Option<String>,
action: Option<String>,
/// The text of its `default=`, as written.
default: Option<String>,
}
/// Every `add_argument(...)` call in a Python source, found past comments and
/// string literals, so a call that is only mentioned is not read as one.
fn script_options(source: &str) -> Vec<ScriptOption> {
let s = source.as_bytes();
let mut options = Vec::new();
let mut i = 0;
while i < s.len() {
match s[i] {
b'#' => i = skip_comment(s, i),
b'"' | b'\'' => i = skip_string(s, i),
c if c.is_ascii_alphabetic() || c == b'_' => {
let start = i;
while i < s.len() && (s[i].is_ascii_alphanumeric() || s[i] == b'_') {
i += 1;
}
let mut open = i;
while open < s.len() && s[open].is_ascii_whitespace() {
open += 1;
}
if &source[start..i] == "add_argument" && s.get(open) == Some(&b'(') {
let (arguments, end) = call_arguments(source, open);
options.push(script_option(&arguments));
i = end;
}
}
_ => i += 1,
}
}
options
}
fn skip_comment(s: &[u8], mut i: usize) -> usize {
while i < s.len() && s[i] != b'\n' {
i += 1;
}
i
}
/// The index just past the string literal whose opening quote is at `i`.
fn skip_string(s: &[u8], i: usize) -> usize {
let quote = s[i];
let triple = s.get(i..i + 3) == Some(&[quote; 3][..]);
let mut j = if triple { i + 3 } else { i + 1 };
while j < s.len() {
if s[j] == b'\\' {
j += 2;
} else if triple && s.get(j..j + 3) == Some(&[quote; 3][..]) {
return j + 3;
} else if !triple && s[j] == quote {
return j + 1;
} else {
j += 1;
}
}
s.len()
}
/// The top-level arguments of the call whose `(` is at `open`, and the index just
/// past its `)`.
fn call_arguments(source: &str, open: usize) -> (Vec<&str>, usize) {
let s = source.as_bytes();
let mut arguments = Vec::new();
let mut depth = 0;
let mut start = open + 1;
let mut i = open + 1;
while i < s.len() {
match s[i] {
b'#' => i = skip_comment(s, i),
b'"' | b'\'' => i = skip_string(s, i),
b'(' | b'[' | b'{' => {
depth += 1;
i += 1;
}
b')' if depth == 0 => {
arguments.push(source[start..i].trim());
arguments.retain(|a| !a.is_empty());
return (arguments, i + 1);
}
b')' | b']' | b'}' => {
depth -= 1;
i += 1;
}
b',' if depth == 0 => {
arguments.push(source[start..i].trim());
start = i + 1;
i += 1;
}
_ => i += 1,
}
}
panic!("an add_argument call at byte {open} never closes")
}
fn script_option(arguments: &[&str]) -> ScriptOption {
let mut option = ScriptOption {
flags: Vec::new(),
required: false,
type_name: None,
action: None,
default: None,
};
for argument in arguments {
let keyword = argument.find('=').filter(|&at| {
let name = argument[..at].trim();
!name.is_empty()
&& name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
&& !argument[at + 1..].starts_with('=')
});
match keyword {
None => option
.flags
.push(string_literal(argument).unwrap_or_else(|| argument.to_string())),
Some(at) => {
let value = argument[at + 1..].trim();
match argument[..at].trim() {
"required" => option.required = value == "True",
"type" => option.type_name = Some(value.to_string()),
"action" => option.action = string_literal(value),
"default" => option.default = Some(value.to_string()),
_ => {}
}
}
}
}
option
}
fn string_literal(text: &str) -> Option<String> {
let text = text
.trim()
.trim_start_matches(|c: char| "rRbBuUfF".contains(c));
["\"\"\"", "'''", "\"", "'"].iter().find_map(|quote| {
text.strip_prefix(quote)
.and_then(|t| t.strip_suffix(quote))
.map(String::from)
})
}
/// The schema type an option's values arrive as, and its items' type for an
/// option that collects a list.
fn argparse_schema_type(
option: &ScriptOption,
) -> std::result::Result<(&'static str, Option<&'static str>), String> {
let scalar = match option.type_name.as_deref() {
None | Some("str") => "string",
Some("int") => "integer",
Some("float") => "number",
Some(other) => return Err(format!("argparse type `{other}`")),
};
match option.action.as_deref() {
None | Some("store") => Ok((scalar, None)),
Some("append") | Some("extend") => Ok(("array", Some(scalar))),
Some("store_true") | Some("store_false") => Ok(("boolean", None)),
Some(other) => Err(format!("argparse action `{other}`")),
}
}
/// The strings a key's probe value reaches the argv as.
fn rendered(value: &Json) -> Vec<String> {
match value {
Json::String(s) => vec![s.clone()],
Json::Number(n) => vec![n.to_string()],
Json::Bool(b) => vec![b.to_string()],
Json::Array(items) => items.iter().flat_map(rendered).collect(),
_ => Vec::new(),
}
}
/// Whether an argv value carries a probe: as given, or as a path under the
/// protocol directory.
fn carries(argument: &str, probe: &str) -> bool {
argument == probe || Path::new(argument).ends_with(probe)
}
/// A frozen-script operator's step, built as far as the argv its `run` hands `uv`:
/// the source of the script that argv names, and each option the argv passes with
/// every value that follows it.
struct ScriptCall {
source: String,
passed: BTreeMap<String, Vec<String>>,
}
fn script_call(op: &str, with: &Json) -> std::result::Result<ScriptCall, String> {
let with: Value = serde_yaml::to_value(with).expect("a JSON value converts to YAML");
let dir = tempfile::tempdir().unwrap();
let argv = script_invocation(op, &with, dir.path())
.map_err(|e| format!("the manifest is refused: {e}"))?;
let [run, script_flag, script, tail @ ..] = argv.as_slice() else {
return Err(format!("the argv is too short to name a script: {argv:?}"));
};
if (run.as_str(), script_flag.as_str()) != ("run", "--script") {
return Err(format!("the argv does not run a script: {argv:?}"));
}
let source = std::fs::read_to_string(script)
.unwrap_or_else(|e| panic!("{op}: reading the script the argv names, {script}: {e}"));
let mut passed: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut current = None;
for argument in tail {
if argument.starts_with("--") {
passed.entry(argument.clone()).or_default();
current = Some(argument);
} else if let Some(option) = current {
passed.get_mut(option).unwrap().push(argument.clone());
} else {
return Err(format!("the argv passes {argument:?} ahead of any option"));
}
}
Ok(ScriptCall { source, passed })
}
/// Which option each schema key reaches in `call`, the step built from `full`, by
/// option: the one whose values carry the key's probe value. A key is found by the
/// value it carries, so a flag that carries none would need this extended; no key
/// reaches one. The faults name each key that cannot be found.
fn options_keys_reach(
schema: &Json,
full: &Json,
call: &ScriptCall,
) -> (BTreeMap<String, String>, Vec<String>) {
let mut key_of = BTreeMap::new();
let mut faults = Vec::new();
let mut seen = BTreeSet::new();
for (key, _) in properties(schema) {
let values = rendered(&full[key]);
if values.is_empty() || !values.iter().all(|v| seen.insert(v.clone())) {
faults.push(format!(
"`{key}`'s probe value {} cannot be told apart in the argv",
full[key]
));
continue;
}
let reached = call
.passed
.iter()
.find(|(_, given)| values.iter().all(|v| given.iter().any(|a| carries(a, v))));
match reached {
Some((option, _)) => {
key_of.insert(option.clone(), key.clone());
}
None => faults.push(format!(
"the schema's `{key}` reaches the script as no option — a manifest setting it \
to {} passes {:?}",
full[key], call.passed
)),
}
}
(key_of, faults)
}
/// The value an option takes when the argv leaves it out, as JSON, `null` for none:
/// its `default=`, or argparse's own without one — `False` for `store_true`, `True`
/// for `store_false`. `Err` names a `default=` this reader cannot evaluate.
fn script_default(option: &ScriptOption, source: &str) -> std::result::Result<Json, String> {
match option.default.as_deref() {
Some(text) => python_value(text, source, 0),
None => Ok(match option.action.as_deref() {
Some("store_true") => json!(false),
Some("store_false") => json!(true),
_ => Json::Null,
}),
}
}
/// A Python literal as JSON — `None`, `True`, `False`, a number or a string — or a
/// name `source` assigns once at module level, read through to the literal it holds.
fn python_value(text: &str, source: &str, depth: usize) -> std::result::Result<Json, String> {
let text = text.trim();
match text {
"None" => return Ok(Json::Null),
"True" => return Ok(json!(true)),
"False" => return Ok(json!(false)),
_ => {}
}
if let Some(literal) = string_literal(text) {
return Ok(Json::String(literal));
}
if let Ok(number) = serde_json::from_str::<serde_json::Number>(&text.replace('_', "")) {
return Ok(Json::Number(number));
}
let is_name = text.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_')
&& text.chars().all(|c| c.is_ascii_alphanumeric() || c == '_');
if !is_name || depth > 8 {
return Err(format!(
"`{text}` is not a literal or a name this check can read"
));
}
let assigned: Vec<&str> = source
.lines()
.filter_map(|line| module_assignment(line, text))
.collect();
match assigned.as_slice() {
[value] => python_value(value, source, depth + 1),
[] => Err(format!(
"the script assigns `{text}` nowhere at module level"
)),
_ => Err(format!(
"the script assigns `{text}` {} times at module level",
assigned.len()
)),
}
}
/// The value `line` assigns to `name` at module level — at column 0, annotated or
/// not — less any trailing comment.
fn module_assignment<'s>(line: &'s str, name: &str) -> Option<&'s str> {
let rest = line.strip_prefix(name)?.trim_start();
let rest = match rest.strip_prefix(':') {
Some(annotated) => &annotated[annotated.find('=')?..],
None => rest,
};
let value = rest.strip_prefix('=').filter(|v| !v.starts_with('='))?;
let s = value.as_bytes();
let mut i = 0;
while i < s.len() && s[i] != b'#' {
i = if s[i] == b'"' || s[i] == b'\'' {
skip_string(s, i)
} else {
i + 1
};
}
Some(value[..i].trim())
}
/// The value an option takes from the argv, as JSON of its argparse type.
fn argv_value(option: Option<&ScriptOption>, values: &[String]) -> Json {
let scalar = |v: &String| match option.and_then(|o| o.type_name.as_deref()) {
Some("int") | Some("float") => serde_json::from_str(v).unwrap_or(json!(v)),
_ => json!(v),
};
match values {
[] => json!(true),
[one] => scalar(one),
many => many.iter().map(scalar).collect(),
}
}
/// `add_argument` calls are read past comments and strings: a call that is only
/// mentioned is not an option.
#[test]
fn script_options_reads_calls_not_mentions() {
let source = r#"
"""Mentions ap.add_argument("--in-docstring") in prose."""
# ap.add_argument("--commented-out")
HELP = 'ap.add_argument("--in-a-string")'
ap.add_argument("--real", required=True, type=int, help="a (parenthesised) note, with commas")
ap.add_argument(
"--listed",
action="append",
help=("two " "parts"),
)
"#;
let options = script_options(source);
let flags: Vec<&str> = options.iter().map(|o| o.flags[0].as_str()).collect();
assert_eq!(flags, ["--real", "--listed"]);
assert!(options[0].required && options[0].type_name.as_deref() == Some("int"));
assert_eq!(options[1].action.as_deref(), Some("append"));
}
/// Each frozen script reads the options its operator's schema reaches, and no
/// other: every option the script declares is reached by a key, or is on
/// [`OPTIONS_ARC_NEVER_PASSES`]; every key reaches the script as an option in the
/// argv `run` builds; an option the script marks `required=True` comes from a key
/// the schema requires; and an option's argparse type is its key's schema type.
///
/// The options are read from the script file the argv names, which is the one `uv`
/// runs. A key is matched to its option by where its probe value lands in that
/// argv, so a key renamed on its way to the script (`columns` reaches
/// `umap_project.py` as one `--column` per entry) needs no table here. A key is
/// found by the value it carries, so a flag that carries none would need this
/// check extended; no key reaches one.
#[test]
fn every_frozen_scripts_options_agree_with_its_operators_schema() {
let catalogue = catalog_names();
let mut faults = Vec::new();
for (op, flag, _) in OPTIONS_ARC_NEVER_PASSES {
if !SCRIPT_OPERATORS.contains(op) {
faults.push(format!("the list of options arc never passes names {op}'s {flag}, and {op} runs no frozen script"));
}
}
for op in SCRIPT_OPERATORS {
assert!(catalogue.contains(&op), "`{op}` is not in the catalogue");
let schema = with_schema(op).expect("every catalogue operator has a schema");
let schema_required = required(&schema);
let full = probe("", &schema);
let call = match script_call(op, &full) {
Ok(call) => call,
Err(e) => {
faults.push(format!("{op}, from a manifest of every schema key: {e}"));
continue;
}
};
let ScriptCall { source, passed } = &call;
let (key_of, unreached) = options_keys_reach(&schema, &full, &call);
faults.extend(unreached.into_iter().map(|f| format!("{op}: {f}")));
let never: BTreeSet<&str> = OPTIONS_ARC_NEVER_PASSES
.iter()
.filter(|(o, _, _)| *o == op)
.map(|(_, flag, _)| *flag)
.collect();
let declared = script_options(source);
for option in &declared {
let name = option.flags.join("/");
let passed_as = option
.flags
.iter()
.find(|f| passed.contains_key(f.as_str()));
let Some(flag) = passed_as else {
if !option.flags.iter().any(|f| never.contains(f.as_str())) {
faults.push(format!(
"{op}: the script reads {name}, which no schema key reaches and which \
is not on the list of options arc never passes"
));
}
continue;
};
if never.contains(flag.as_str()) {
faults.push(format!(
"{op}: {flag} is on the list of options arc never passes, but arc passes it"
));
}
let Some(key) = key_of.get(flag.as_str()) else {
continue;
};
if option.required && !schema_required.contains(key) {
faults.push(format!(
"{op}: the script marks {flag} required=True, but the schema does not require `{key}`"
));
}
let prop = &schema["properties"][key];
let schema_type = (prop["type"].as_str(), prop["items"]["type"].as_str());
match argparse_schema_type(option) {
Ok((t, items)) if (Some(t), items) == schema_type => {}
Ok((t, items)) => faults.push(format!(
"{op}: the script reads {flag} as {t}{}, but the schema types `{key}` as {}{}",
items.map(|i| format!(" of {i}")).unwrap_or_default(),
schema_type.0.unwrap_or("nothing"),
schema_type.1.map(|i| format!(" of {i}")).unwrap_or_default(),
)),
Err(what) => faults.push(format!(
"{op}: the script reads {flag} with {what}, which this check cannot match to a schema type"
)),
}
}
for flag in passed.keys() {
if !declared.iter().any(|o| o.flags.iter().any(|f| f == flag)) {
faults.push(format!(
"{op}: arc passes {flag}, which the script does not declare"
));
}
}
for flag in &never {
if !declared.iter().any(|o| o.flags.iter().any(|f| f == flag)) {
faults.push(format!(
"{op}: the list of options arc never passes names {flag}, which the script does not declare"
));
}
}
}
fail_on(
&faults,
"a frozen script and its operator's schema disagree",
);
}
/// A `default=` is read through the name it gives to the literal that name is
/// assigned at module level, past a comment; a name assigned twice, or nowhere, is
/// not read.
#[test]
fn script_defaults_are_read_through_module_names() {
let source = r#"
LIMIT = 15 # the fallback
RATE: float = 0.1
AGENT = "a (b; c)"
TWICE = 1
TWICE = 2
ap.add_argument("--limit", type=int, default=LIMIT)
ap.add_argument("--rate", type=float, default=RATE)
ap.add_argument("--agent", default=AGENT)
ap.add_argument("--size", type=int, default=200)
ap.add_argument("--flag", action="store_true")
ap.add_argument("--given")
ap.add_argument("--twice", default=TWICE)
ap.add_argument("--nowhere", default=NOWHERE)
"#;
let read: Vec<std::result::Result<Json, String>> = script_options(source)
.iter()
.map(|option| script_default(option, source))
.collect();
assert_eq!(
read[..6],
[
Ok(json!(15)),
Ok(json!(0.1)),
Ok(json!("a (b; c)")),
Ok(json!(200)),
Ok(json!(false)),
Ok(Json::Null),
]
);
assert!(
read[6].as_ref().is_err_and(|e| e.contains("2 times")),
"{:?}",
read[6]
);
assert!(
read[7].as_ref().is_err_and(|e| e.contains("nowhere")),
"{:?}",
read[7]
);
}
/// Each value a frozen-script operator's step takes when its key is left out is the
/// `default` its schema states for the key, and a key whose step takes none states
/// none. What the step takes is read where it is decided: the value the argv passes
/// for the key's option when the key is left out, or else the script's own default
/// for that option, read from the script the argv names and through a module-level
/// name when the `default=` gives one. Every optional key is left out at once, from
/// a manifest of the required keys. A required key is always set, so a `default`
/// stated for one is a fault as well.
#[test]
fn every_default_a_frozen_script_takes_is_the_default_its_schema_states() {
let mut faults = Vec::new();
for op in SCRIPT_OPERATORS {
let schema = with_schema(op).expect("every catalogue operator has a schema");
let schema_required = required(&schema);
let full = probe("", &schema);
let required_only: Json = full
.as_object()
.expect("a probe of a schema object is an object")
.iter()
.filter(|(key, _)| schema_required.contains(*key))
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<serde_json::Map<_, _>>()
.into();
let (every_key, left_out) =
match (script_call(op, &full), script_call(op, &required_only)) {
(Ok(every_key), Ok(left_out)) => (every_key, left_out),
(Err(e), _) => {
faults.push(format!("{op}, from a manifest of every schema key: {e}"));
continue;
}
(_, Err(e)) => {
faults.push(format!("{op}, from a manifest of the required keys: {e}"));
continue;
}
};
let (key_of, _) = options_keys_reach(&schema, &full, &every_key);
let declared = script_options(&left_out.source);
for (key, prop) in properties(&schema) {
let stated = prop.get("default");
if schema_required.contains(key) {
if let Some(stated) = stated {
faults.push(format!(
"{op}: the schema requires `{key}` and states {stated} as its default"
));
}
continue;
}
let Some(option) = key_of.iter().find(|(_, k)| *k == key).map(|(o, _)| o) else {
faults.push(format!(
"{op}: `{key}` reaches no option this check can find, so the value a \
step takes when it is left out is unread"
));
continue;
};
let script_option = declared.iter().find(|o| o.flags.contains(option));
let (taken, from) = match (left_out.passed.get(option), script_option) {
(Some(values), _) => (
argv_value(script_option, values),
format!("arc passing {option} {values:?}"),
),
(None, Some(script_option)) => {
match script_default(script_option, &left_out.source) {
Ok(taken) => (
taken,
match &script_option.default {
Some(text) => format!("the script's {option} default={text}"),
None => {
format!("the script's {option}, which gives no default=")
}
},
),
Err(e) => {
faults.push(format!(
"{op}: the default of `{key}`'s option {option} cannot be read: {e}"
));
continue;
}
}
}
(None, None) => {
faults.push(format!(
"{op}: the script does not declare {option}, which `{key}` reaches"
));
continue;
}
};
match stated {
None if taken.is_null() => {}
None => faults.push(format!(
"{op}: a step that leaves `{key}` out takes {taken}, from {from}, and \
the schema states no default for `{key}`"
)),
Some(stated) if *stated == taken => {}
Some(stated) => faults.push(format!(
"{op}: the schema states {stated} as `{key}`'s default, and a step that \
leaves `{key}` out takes {taken}, from {from}"
)),
}
}
}
fail_on(
&faults,
"a value a frozen script takes when its key is left out differs from the default \
its schema states",
);
}
/// The configuration a `with:` block is read into, written out, or why it is
/// refused: the typed deserialize a manifest passes.
fn configured(op: &str, with: &Json) -> std::result::Result<String, String> {
fn read_as<T: serde::de::DeserializeOwned + std::fmt::Debug>(
with: &Json,
) -> std::result::Result<String, String> {
let with: Value = serde_yaml::to_value(with).expect("a JSON value converts to YAML");
serde_yaml::from_value::<T>(with)
.map(|configuration| format!("{configuration:?}"))
.map_err(|e| e.to_string())
}
match op {
"parquet_export" => read_as::<ParquetExportConfig>(with),
#[cfg(feature = "http-fetch")]
"http_fetch" => read_as::<HttpFetchConfig>(with),
#[cfg(feature = "opendal")]
"opendal_fetch" => read_as::<OpendalFetchConfig>(with),
#[cfg(feature = "http-fetch")]
"html_link_discover" => read_as::<HtmlLinkDiscoverConfig>(with),
"archive_extract" => read_as::<ArchiveExtractConfig>(with),
"datapackage_describe" => read_as::<DatapackageDescribeConfig>(with),
"finetype_validate" => read_as::<FinetypeValidateConfig>(with),
"splink_resolve" => read_as::<SplinkResolveConfig>(with),
"gleif_ra_fetch" => read_as::<GleifRaFetchConfig>(with),
"umap_project" => read_as::<UmapProjectConfig>(with),
"text_embed" => read_as::<TextEmbedConfig>(with),
"uv" => read_as::<UvConfig>(with),
"ducklake_publish" => read_as::<ducklake_publish::DucklakePublishConfig>(with),
other => panic!(
"operator `{other}` is in the catalogue, but configured does not name its \
configuration struct — add it there"
),
}
}
/// `full` with the key `key` of the object at `path` left out.
fn without_key_at(full: &Json, path: &str, key: &str) -> Json {
let mut object = if path.is_empty() {
full.clone()
} else {
full.pointer(&format!("/{}", path.replace('.', "/")))
.expect("the path is in the manifest")
.clone()
};
object
.as_object_mut()
.expect("a schema object's probe is an object")
.remove(key);
with_object_at(full, path, object)
}
/// Each value an operator's configuration takes when its key is left out is the
/// `default` its schema states for the key, and a key whose configuration takes
/// none states none. It is read by running the typed deserialize a manifest passes:
/// a manifest that leaves a key out is read into the configuration a manifest
/// setting it to its stated default is read into. With no default stated, it is
/// read into the configuration a manifest setting it to null is, or, for a list or
/// a map, an empty one: an unset value, and an empty list or map, is no value
/// taken. A frozen-script operator's values are taken by its script, and the test
/// above holds them.
#[test]
fn every_default_a_configuration_takes_is_the_default_its_schema_states() {
let mut faults = Vec::new();
for op in catalog_names() {
if SCRIPT_OPERATORS.contains(&op) {
continue;
}
let schema = with_schema(op).expect("every catalogue operator has a schema");
let full = probe("", &schema);
for (path, object) in schema_objects(&schema) {
let object_required = required(object);
for (key, prop) in properties(object) {
let name = joined(&path, key);
let stated = prop.get("default");
if object_required.contains(key) {
if let Some(stated) = stated {
faults.push(format!(
"{op}: the schema requires `{name}` and states {stated} as its default"
));
}
continue;
}
let left_out = configured(op, &without_key_at(&full, &path, key));
let set_to =
|value: Json| configured(op, &with_key_at(&full, &path, key, value));
match (stated, &left_out) {
(Some(stated), Err(e)) => faults.push(format!(
"{op}: the schema states {stated} as `{name}`'s default, and a manifest \
that leaves `{name}` out is refused: {e}"
)),
(Some(stated), Ok(read)) => {
let set = set_to(stated.clone());
if set != left_out {
faults.push(format!(
"{op}: the schema states {stated} as `{name}`'s default, and a \
manifest that leaves `{name}` out is read as {read}, where one \
setting it to {stated} is read as {}",
set.unwrap_or_else(|e| format!("refused: {e}"))
));
}
}
(None, Err(_)) => {}
(None, Ok(read)) => {
let unset = [
Some(Json::Null),
(prop["type"] == "array").then(|| json!([])),
(prop["type"] == "object").then(|| json!({})),
];
if !unset.into_iter().flatten().any(|u| set_to(u) == left_out) {
faults.push(format!(
"{op}: a manifest that leaves `{name}` out is read as {read}, \
which no manifest setting it to null or to nothing is, so it \
takes a value when left out, and the schema states no default \
for `{name}`"
));
}
}
}
}
}
}
fail_on(
&faults,
"a value a configuration takes when its key is left out differs from the default its \
schema states",
);
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Subprocess substrate — the shared spawn/wait/error-map for operators that wrap
// an external process: a `uv run` Python script (splink_resolve, gleif_ra_fetch)
// or a CLI (datapackage_describe → finetype). One place for the retry taxonomy so
// each op doesn't hand-roll a Command block with divergent error handling.
// (Wired by those operators in later increments — allow(dead_code) until then.)
// ─────────────────────────────────────────────────────────────────────────────
/// How a subprocess operator handles its child's output.
enum OutputMode {
/// Inherit the terminal — stream child stdout/stderr live (e.g. `splink_resolve`'s
/// coverage tables). Honours the step timeout via `wait_with_timeout`.
Inherit,
/// Capture stdout+stderr into `StepOutput` — for ops that parse stdout
/// (`datapackage_describe`) or surface child stderr. `output()` drains the pipes
/// (no fill-up deadlock); the timeout is not enforced here (parity with the
/// capturing command path today).
Capture,
}
/// Spawn `program args…` in `ctx.dir` with `ctx.env`, mapping failures to the engine's
/// retry taxonomy: a **spawn** failure (missing binary — deterministic) is a
/// NON-retryable [`Error::StepExecution`] so a bad binary never burns 3 attempts of a
/// 4 h job; a **non-zero exit** is a retryable [`Error::StepFailed`]; a **deadline** is
/// [`Error::StepTimeout`] (retryable). `name` labels timeout/exec errors (the runner
/// rewrites only `StepFailed.step`).
fn run_process(
program: &str,
args: &[String],
ctx: &OpContext,
mode: OutputMode,
name: &str,
) -> Result<StepOutput> {
use std::process::{Command, Stdio};
let mut cmd = Command::new(program);
cmd.args(args).current_dir(ctx.dir).envs(ctx.env);
match mode {
OutputMode::Inherit => {
cmd.stdout(Stdio::inherit()).stderr(Stdio::inherit());
let mut child = cmd.spawn().map_err(|e| Error::StepExecution {
step: name.to_string(),
source: e,
})?;
let status = crate::engine::wait_with_timeout(&mut child, ctx.timeout, name)?;
if !status.success() {
return Err(Error::StepFailed {
step: String::new(), // runner rewrites with the step name
code: status.code().unwrap_or(1),
stderr: String::new(),
});
}
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
OutputMode::Capture => {
let out = cmd.output().map_err(|e| Error::StepExecution {
step: name.to_string(),
source: e,
})?;
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
if !out.status.success() {
return Err(Error::StepFailed {
step: String::new(),
code: out.status.code().unwrap_or(1),
stderr,
});
}
Ok(StepOutput {
stderr,
stdout: Some(String::from_utf8_lossy(&out.stdout).to_string()),
report: None,
})
}
}
}
/// `["run", "--script", <script>, <extra…>]` — the shared `uv run --script` invocation
/// for the uv-backed Python operators. Factored out so its arg order is unit-testable.
fn uv_run_args(script: &str, extra: &[String]) -> Vec<String> {
let mut a = vec![
"run".to_string(),
"--script".to_string(),
script.to_string(),
];
a.extend_from_slice(extra);
a
}
/// Materialize an `include_str!`-embedded operator script to a version-stamped cache
/// under the temp dir (write-if-changed), returning the path to run. Pinning the
/// script into the binary means `op@<ver>` addresses **exact** bytes — a script change
/// is a version bump + rebuild, never a silent edit. This is the reproducibility
/// contract the old "call the script by relative path" step lacked.
fn materialize_frozen_script(name: &str, version: &str, bytes: &str) -> Result<PathBuf> {
let dir = std::env::temp_dir().join(format!("arcform-op-{}-{}", name, version));
std::fs::create_dir_all(&dir)
.map_err(|e| fetch_failed(format!("{}: cache dir {}: {}", name, dir.display(), e)))?;
let path = dir.join(format!("{}.py", name));
write_script_if_changed(&path, bytes)
.map_err(|e| fetch_failed(format!("{}: write {}: {}", name, path.display(), e)))?;
Ok(path)
}
/// Put `bytes` at `path` unless the file there already holds exactly them.
///
/// The cache directory is shared — by every run of `arc` on the machine that uses the
/// operator, and by the threads of one test binary — and a caller that finds the
/// script already there reads it without taking any lock. So the write goes through
/// [`crate::edit::write_atomic`]: a temp file named for the writer, beside the target,
/// renamed over it. A reader then opens the whole old script or the whole new one. It
/// cannot open the file empty, as it could between the truncate and the write of
/// `std::fs::write`, and a reader that already has the old script open keeps reading
/// the old bytes, since the rename gives the path a new file instead of changing the
/// one it had.
fn write_script_if_changed(path: &Path, bytes: &str) -> Result<()> {
let up_to_date = std::fs::read_to_string(path)
.map(|on_disk| on_disk == bytes)
.unwrap_or(false);
if !up_to_date {
crate::edit::write_atomic(path, bytes.as_bytes())?;
}
Ok(())
}
#[cfg(test)]
mod frozen_script_write_tests {
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
/// Two scripts of different bytes and the same length, each big enough that a write
/// in place is a window a reader can land in rather than a single instruction.
fn script(fill: u8) -> String {
format!("# {}\n", char::from(fill)).repeat(128 * 1024)
}
/// The directory `materialize_frozen_script` made for a test's own operator name,
/// removed when the test ends. The two tests that go through `materialize_frozen_script`
/// itself, and so through the real temp directory, use a name of their own so that
/// no other test's cache is written.
struct CacheDir(PathBuf);
impl Drop for CacheDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
/// A reader that opens the script's path while a writer is replacing it reads the
/// whole script — the one before the write or the one after — and reads neither an
/// empty file nor part of one.
///
/// The reader spins on `std::fs::read` for as long as the writer swaps between two
/// scripts, and every read has to be one of the two in full. With the write made in
/// place (`std::fs::write`), a read that lands between the truncate and the write
/// returns `""`, and one that lands inside the write returns part of the script.
///
/// The writer is `materialize_frozen_script`, not the function under it, so the write
/// made in place is caught wherever in the two it is made.
#[test]
fn a_reader_of_a_script_being_replaced_reads_the_whole_of_one_version() {
let name = format!("frozen_write_probe_reader_{}", std::process::id());
let (a, b) = (script(b'a'), script(b'b'));
let path = materialize_frozen_script(&name, "0", &a).unwrap();
let _cleanup = CacheDir(path.parent().unwrap().to_path_buf());
let writing = AtomicBool::new(true);
let torn = std::thread::scope(|s| {
let reader = s.spawn(|| {
let mut torn = Vec::new();
while writing.load(Ordering::Acquire) {
let seen = std::fs::read(&path).expect("the script is there to read");
if seen != a.as_bytes() && seen != b.as_bytes() {
torn.push(seen.len());
}
}
torn
});
for i in 0..200 {
let written =
materialize_frozen_script(&name, "0", if i % 2 == 0 { &b } else { &a })
.unwrap();
assert_eq!(
written, path,
"the path handed back is the same one each time"
);
}
writing.store(false, Ordering::Release);
reader.join().unwrap()
});
assert!(
torn.is_empty(),
"a reader read {} times a script that was neither version in full; lengths \
seen: {:?}, against {} in each version",
torn.len(),
&torn[..torn.len().min(8)],
a.len()
);
}
/// A write of different bytes gives the path a different file. A reader that had the
/// old file open still reads the old bytes whole, and a reader that opens the path
/// after reads the new ones.
#[cfg(unix)]
#[test]
fn a_write_of_different_bytes_gives_the_path_a_new_file_and_leaves_the_old_one_whole() {
use std::io::Read;
use std::os::unix::fs::MetadataExt;
let name = format!("frozen_write_probe_inode_{}", std::process::id());
let (old, new) = (script(b'o'), script(b'n'));
let path = materialize_frozen_script(&name, "0", &old).unwrap();
let _cleanup = CacheDir(path.parent().unwrap().to_path_buf());
let mut held = std::fs::File::open(&path).unwrap();
let old_inode = held.metadata().unwrap().ino();
materialize_frozen_script(&name, "0", &new).unwrap();
assert_ne!(
std::fs::metadata(&path).unwrap().ino(),
old_inode,
"the path names a different file after a write of different bytes"
);
let mut still_old = String::new();
held.read_to_string(&mut still_old).unwrap();
assert_eq!(
still_old, old,
"a reader that had the old file open reads the old bytes whole"
);
assert_eq!(std::fs::read_to_string(&path).unwrap(), new);
}
/// A script whose bytes on disk are those asked for is left as it is: not written
/// again, so its modification time and its file are the ones it had.
#[test]
fn a_script_already_holding_the_bytes_is_not_written_again() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("op.py");
write_script_if_changed(&path, "print('x')\n").unwrap();
// An hour back, so a rewrite is a visible change and not a tie on the clock's grain.
let hour_ago = std::time::SystemTime::now() - Duration::from_secs(3600);
filetime::set_file_mtime(&path, filetime::FileTime::from_system_time(hour_ago)).unwrap();
let before = std::fs::metadata(&path).unwrap().modified().unwrap();
write_script_if_changed(&path, "print('x')\n").unwrap();
assert_eq!(
std::fs::metadata(&path).unwrap().modified().unwrap(),
before,
"a script already holding the bytes keeps its modification time"
);
}
/// After a write the directory holds the script and no other file — none left
/// under the name the write went through. Checked after the first write and after a
/// write over a script of other bytes.
#[test]
fn a_write_leaves_the_script_and_no_file_under_another_name() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("op.py");
for bytes in ["print(1)\n", "print(2)\n"] {
write_script_if_changed(&path, bytes).unwrap();
let names: Vec<_> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.collect();
assert_eq!(
names,
["op.py"],
"the directory holds the script and no file under another name"
);
assert_eq!(std::fs::read_to_string(&path).unwrap(), bytes);
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// parquet_export (activate) — the terminal Parquet output as a first-class asset.
//
// Replaces the opaque `COPY … (COMPRESSION zstd)` step whose SQL the engine can't
// introspect, so it becomes a graph island that can silently skip and ship a stale
// artifact. By declaring `reads: [input]`, the export sits *in* the graph downstream
// of the table it exports — no manual `depends_on`, no silent stale-ship.
// ─────────────────────────────────────────────────────────────────────────────
struct ParquetExport;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct ParquetExportConfig {
/// Source table or view in the pipeline DB to export.
input: String,
/// Destination Parquet path, relative to the manifest directory.
dest: String,
/// Parquet codec. Defaults to `zstd`.
#[serde(default = "default_zstd")]
compression: String,
/// Optional Parquet row-group size.
#[serde(default)]
row_group_size: Option<u64>,
/// Optional `ORDER BY` clause applied to the export.
#[serde(default)]
order_by: Option<String>,
/// Optional Parquet **key-value metadata**, written into the file's footer.
///
/// Keys and values are both strings and are written as their **UTF-8 bytes**.
/// Parquet's footer map is untyped bytes, so DuckDB reads them back as `BLOB`:
/// `SELECT decode(key), decode(value) FROM parquet_kv_metadata('f.parquet')`
/// recovers the strings. **`value::VARCHAR` does not** — casting a `BLOB` to
/// `VARCHAR` yields DuckDB's escaped rendering (`"` becomes `\x22`), not the
/// text that was written. `decode()` is the read-back.
///
/// Deliberately untyped: this operator carries whatever a protocol wants
/// stamped and takes no view on what the keys mean.
///
/// A `BTreeMap`, so the entries are emitted in sorted key order however the
/// `with:` block lists them. Parquet stores the map as an ordered list and
/// DuckDB writes it in the order given, so an unordered map here would move
/// the output bytes between runs. See [`parquet_export_sql`].
#[serde(default)]
metadata: BTreeMap<String, String>,
}
fn default_zstd() -> String {
"zstd".to_string()
}
impl ParquetExportConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("parquet_export: invalid `with:` config: {}", e))
})
}
}
/// A DuckDB single-quoted string literal: wrap in `'` and double any interior `'`.
///
/// That is the whole escape — a plain `'…'` literal in DuckDB processes no backslash
/// escapes (unlike `E'…'`), so a backslash, a newline and a `"` all pass through as
/// themselves. Verified against DuckDB 1.5.4 for `\`, LF and `"`.
fn sql_string_literal(s: &str) -> String {
format!("'{}'", s.replace('\'', "''"))
}
/// Build the `COPY … TO … (…)` statement a `parquet_export` step executes.
///
/// Split out of [`Operator::run`] so the statement — and above all its option list,
/// which is where the output bytes are decided — is testable without a DuckDB
/// connection or a written file.
///
/// **Byte-reproducibility.** An export declaring no `metadata` emits no
/// `KV_METADATA` option at all, so its statement is character-identical to the one
/// this operator built before key-value metadata existed and its output bytes do not
/// move. An export that does declare metadata necessarily changes the footer and so
/// the file's hash; the `order_by` clause that makes an export reproducible still
/// does, because the entries are emitted in sorted key order. An empty map takes the
/// no-metadata path for the same reason it must: DuckDB rejects `KV_METADATA {}` as a
/// syntax error.
fn parquet_export_sql(cfg: &ParquetExportConfig, dest: &Path) -> String {
let order = cfg
.order_by
.as_deref()
.map(|o| format!(" ORDER BY {}", o))
.unwrap_or_default();
let mut opts = format!("FORMAT parquet, COMPRESSION {}", cfg.compression);
if let Some(rg) = cfg.row_group_size {
opts.push_str(&format!(", ROW_GROUP_SIZE {}", rg));
}
if !cfg.metadata.is_empty() {
let entries: Vec<String> = cfg
.metadata
.iter()
.map(|(k, v)| format!("{}: {}", sql_string_literal(k), sql_string_literal(v)))
.collect();
opts.push_str(&format!(", KV_METADATA {{{}}}", entries.join(", ")));
}
format!(
"COPY (SELECT * FROM {input}{order}) TO '{dest}' ({opts});",
input = cfg.input,
order = order,
dest = dest.display(),
opts = opts,
)
}
impl Operator for ParquetExport {
fn name(&self) -> &'static str {
"parquet_export"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = ParquetExportConfig::parse(with)?;
let mut assets = OpAssets::default();
assets.record_reads(cfg.input.clone(), crate::asset_kind::AssetKind::Table);
assets.record_produces(cfg.dest.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = ParquetExportConfig::parse(with)?;
let dest = ctx.dir.join(&cfg.dest);
if let Some(parent) = dest.parent() {
let _ = std::fs::create_dir_all(parent);
}
let sql = parquet_export_sql(&cfg, &dest);
let conn = duckdb::Connection::open(ctx.db_path).map_err(|e| Error::StepFailed {
step: String::new(),
code: 1,
stderr: format!("parquet_export: open db {}: {}", ctx.db_path.display(), e),
})?;
conn.execute_batch(&sql).map_err(|e| Error::StepFailed {
step: String::new(),
code: 1,
stderr: format!("parquet_export: {}", e),
})?;
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// http_fetch (ingress) — the curl/wget of the catalog: an authenticated GET
// streamed atomically to `out`, with a **content-freshness contract**.
//
// Built on **ureq** — already an arcform dependency, blocking, ~0 transitive
// weight, http/https only. Resilience (retry + backoff) comes from the ENGINE's
// step-retry loop, which already wraps every op run via `defaults.retry`. A
// default User-Agent is always set (gov registries — SEC — 403 a missing UA) and
// any `headers` the Protocol supplies are layered on top.
//
// The one thing `command: curl` can't give cleanly, and the reason this operator
// is worth owning: the **freshness contract** ([`crate::ingress_meta`]). Every
// fetch records the remote identity (ETag / Last-Modified / sha256) in a sidecar
// `<out>.arcmeta`; the next run replays it as an `If-None-Match` conditional
// request, so an unchanged 127 MB remote answers `304` and is not re-downloaded.
// The validator recorded is the first hop's where that hop offers one, and the
// last hop's otherwise — see the redirect loop in `run`.
// Paired with the `fresh` precondition (which HEAD-probes the same sidecar), a
// step re-runs — and propagates downstream — only when the remote actually
// changed: content-addressed ingress, not the clock-based mtime `modified_after`.
// This is the workhorse that retires `fetch_edgar`/`fetch_gleif` and the SEC fetch.
//
// REACHABILITY (2026-07-24). This operator, `html_link_discover`, and the `fresh`
// precondition are the crate's only ureq users, and ureq is the only runtime-linked
// consumer of the rustls/ring TLS stack. None of them is reachable through the
// published library surface — `arcform::spec` exports the spec loader alone (src/lib.rs);
// the engine, the operator catalog, and this operator are all private. A pipeline is
// only ever run through the `arc` CLI. So a crate linking arc with
// `default-features = false` — brightfield's desktop shell, which wants only the spec
// loader — cannot call this path, yet before this gating still compiled ureq (and
// rustls + ring) into its binary as dead weight. The path is therefore gated behind
// `http-fetch` (Cargo.toml), which `cli` pulls in; a no-CLI consumer drops ureq and
// its whole TLS stack from the build. (DuckDB still links in via the private engine,
// which is a separate, larger surface — out of scope here.)
// ─────────────────────────────────────────────────────────────────────────────
#[cfg(feature = "http-fetch")]
struct HttpFetch;
#[cfg(feature = "http-fetch")]
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct HttpFetchConfig {
/// Source URL (http/https).
url: String,
/// Destination path, relative to the manifest directory. Written atomically.
out: String,
/// Extra request headers. A default `User-Agent` is set unless overridden here.
#[serde(default)]
headers: BTreeMap<String, String>,
/// The artifact's SHA-256, pinned by the Protocol. Naming the bytes is the one
/// thing that lets the fetch skip revalidation: a run cannot disagree with an
/// origin it never asks, and the manifest has said which bytes it wants.
#[serde(default)]
sha256: Option<String>,
}
#[cfg(feature = "http-fetch")]
impl HttpFetchConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("http_fetch: invalid `with:` config: {}", e))
})
}
/// The pinned digest, normalised, or a manifest error naming the bad value — a
/// pin that cannot match anything is a typo, and a typo that reads as "always
/// re-download" would be the worst of both.
fn pinned_digest(&self) -> Result<Option<String>> {
match self.sha256.as_deref() {
None => Ok(None),
Some(raw) => crate::fetch_cache::parse_digest(raw)
.map(Some)
.ok_or_else(|| {
Error::ManifestValidation(format!(
"http_fetch: `sha256: {raw}` is not a SHA-256 (expected 64 hex characters)"
))
}),
}
}
/// Whether the request carries a credential. The shared cache is keyed by URL, so
/// it cannot tell one credential's bytes from another's — a credentialed fetch
/// therefore neither reads from it nor writes to it. Same two headers ureq drops
/// across a redirect, and for the same reason.
fn is_credentialed(&self) -> bool {
self.headers
.keys()
.any(|k| k.eq_ignore_ascii_case("authorization") || k.eq_ignore_ascii_case("cookie"))
}
}
/// Header carrying the validator of the content a redirect points at, as opposed to
/// the validator of the redirect response itself.
#[cfg(feature = "http-fetch")]
const LINKED_ETAG: &str = "X-Linked-ETag";
/// Redirect hops followed before the fetch gives up. Matches ureq's own default.
#[cfg(feature = "http-fetch")]
const MAX_REDIRECTS: u32 = 5;
/// What a hop of the redirect chain said about the artifact. The fetch keeps the
/// first hop's where that hop offers a validator, because that is the hop `url`
/// addresses; where it offers none, the last hop's, and the conditional request is
/// forwarded down the chain to reach it.
#[cfg(feature = "http-fetch")]
#[derive(Default)]
struct RemoteIdentity {
etag: Option<String>,
last_modified: Option<String>,
content_sha256: Option<String>,
}
#[cfg(feature = "http-fetch")]
impl RemoteIdentity {
/// True when the hop offered nothing a conditional request could be built from.
fn offers_nothing(&self) -> bool {
self.etag.is_none() && self.last_modified.is_none() && self.content_sha256.is_none()
}
fn read(resp: &ureq::Response) -> Self {
let linked = resp.header(LINKED_ETAG).map(str::to_string);
Self {
content_sha256: linked.as_deref().and_then(sha256_validator),
// `ETag` is the response's own validator and is preferred where the
// origin sends one; a redirect that sends only `LINKED_ETAG` has that
// header as its sole offered validator.
etag: resp
.header("ETag")
.map(str::to_string)
.or_else(|| linked.clone()),
last_modified: resp.header("Last-Modified").map(str::to_string),
}
}
}
/// The digest inside a validator that is a bare 64-character hex string, unquoted
/// and lowercased. `None` for any other shape.
#[cfg(feature = "http-fetch")]
fn sha256_validator(validator: &str) -> Option<String> {
let v = validator.trim().trim_start_matches("W/").trim_matches('"');
(v.len() == 64 && v.bytes().all(|b| b.is_ascii_hexdigit())).then(|| v.to_ascii_lowercase())
}
/// Resolve a `Location` against the URL that issued it, so a relative redirect
/// target is followed the way ureq would have followed it.
#[cfg(feature = "http-fetch")]
fn join_location(base: &str, location: &str) -> Option<String> {
Some(url::Url::parse(base).ok()?.join(location).ok()?.to_string())
}
/// The remote is unchanged: the artifact and its sidecar stay as they are.
#[cfg(feature = "http-fetch")]
fn unchanged() -> StepOutput {
StepOutput {
stderr: String::new(),
stdout: None,
report: None,
}
}
/// Map any fetch/IO failure to the retryable `StepFailed` so the engine's
/// step-retry loop re-attempts it with backoff.
fn fetch_failed(msg: String) -> Error {
Error::StepFailed {
step: String::new(), // runner overwrites with the step name
code: 1,
stderr: msg,
}
}
/// Best-effort wall-clock stamp on a sidecar — audit only, and the one field a
/// cached run cannot reproduce from an uncached one.
#[cfg(feature = "http-fetch")]
fn now_unix() -> Option<u64> {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.map(|d| d.as_secs())
}
/// A record pairing **this step's** addressing — its `url`, its request headers — with
/// `entry`'s content identity, which is the half that was verified against bytes.
///
/// It is what crosses the boundary in each direction, and the direction is why the
/// split matters.
///
/// Reading, it is the sidecar this Protocol writes for bytes the shared cache
/// supplied, and the equality is the whole of local≡cached parity: the record a cached
/// run leaves behind is the record the transfer would have left, field for field, down
/// to `fetched_unix`, which is a clock and differs between any two runs.
///
/// Writing, `entry` is a `<out>.arcmeta` — plain YAML inside a Protocol directory,
/// which anything that can write a file can author, and `url` is as authored as
/// `sha256` is. Re-addressing it here is what stops a Protocol seeding the shared store
/// for a URL its step never named.
#[cfg(feature = "http-fetch")]
fn adopt(
cfg: &HttpFetchConfig,
entry: &crate::ingress_meta::FetchMeta,
) -> crate::ingress_meta::FetchMeta {
crate::ingress_meta::FetchMeta {
url: cfg.url.clone(),
request_headers: cfg.headers.clone(),
etag: entry.etag.clone(),
last_modified: entry.last_modified.clone(),
sha256: entry.sha256.clone(),
content_sha256: entry.content_sha256.clone(),
fetched_unix: now_unix(),
}
}
/// The sidecar for a **pinned** artifact the fetch did not transfer. Where the shared
/// entry is the pinned bytes it also carries the validators, so a later run of the
/// same Protocol *without* the pin still has something to revalidate with; where it
/// is not, the digest is all that is known and the sidecar says so.
#[cfg(feature = "http-fetch")]
fn pinned_meta(
cfg: &HttpFetchConfig,
digest: &str,
entry: Option<&crate::ingress_meta::FetchMeta>,
) -> crate::ingress_meta::FetchMeta {
match entry.filter(|e| e.sha256.eq_ignore_ascii_case(digest)) {
Some(e) => adopt(cfg, e),
None => crate::ingress_meta::FetchMeta {
url: cfg.url.clone(),
request_headers: cfg.headers.clone(),
sha256: digest.to_string(),
fetched_unix: now_unix(),
..Default::default()
},
}
}
/// Copy a cache entry into the Protocol and leave the sidecar an uncached fetch would
/// have left. The one line of output is the point of the feature being visible: a run
/// that says nothing is indistinguishable from one that downloaded 127 MB again.
#[cfg(feature = "http-fetch")]
fn serve_from_cache(
cache: &crate::fetch_cache::FetchCache,
cfg: &HttpFetchConfig,
meta: &crate::ingress_meta::FetchMeta,
out: &Path,
) -> Result<StepOutput> {
cache.materialise(meta, out).map_err(|e| {
fetch_failed(format!(
"http_fetch: {} from the shared cache: {}",
cfg.url, e
))
})?;
let _ = crate::ingress_meta::write(out, meta);
{
use owo_colors::OwoColorize;
eprintln!(
"{} {} from the shared fetch cache — no transfer",
"cached:".dimmed(),
cfg.out
);
}
Ok(unchanged())
}
/// The remote is unchanged. Either the Protocol already holds the bytes, in which
/// case the shared cache is seeded from them the first time so the *next* Protocol
/// naming this URL transfers nothing — or the cache holds them and the Protocol does
/// not, in which case they are copied in.
///
/// A `304` with neither is a remote answering "unchanged" about bytes nobody has:
/// nothing was downloaded, so nothing changes here either.
#[cfg(feature = "http-fetch")]
fn keep_or_materialise(
cfg: &HttpFetchConfig,
out: &Path,
prior: Option<&crate::ingress_meta::FetchMeta>,
shared: Option<&crate::ingress_meta::FetchMeta>,
cache: Option<&crate::fetch_cache::FetchCache>,
) -> Result<StepOutput> {
if let Some(p) = prior {
if let Some(c) = cache
&& !c.holds(&cfg.url)
{
// The store files a locator under the record's own `url`, and `p` is this
// Protocol's `<out>.arcmeta`. `adopt` re-addresses it to the URL the step
// named, so what a Protocol can seed is bounded by what it fetched — and
// the guard above then asks about the URL the write targets, so an entry
// that is already there is not refiled on every run. `store` hashes what it
// is given and refuses bytes that do not match the key, so a Protocol whose
// copy has rotted cannot seed the cache with it.
let _ = c.store(&adopt(cfg, p), out);
}
return Ok(unchanged());
}
match (cache, shared) {
(Some(c), Some(entry)) => serve_from_cache(c, cfg, &adopt(cfg, entry), out),
_ => Ok(unchanged()),
}
}
#[cfg(feature = "http-fetch")]
impl Operator for HttpFetch {
fn name(&self) -> &'static str {
"http_fetch"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = HttpFetchConfig::parse(with)?;
// A pin that cannot be a digest is a manifest error, and this is the
// load-time gate — refusing it here means a run never starts on a pin that
// could not have matched.
cfg.pinned_digest()?;
// The network source is not a graph node; only the local artifact is.
let mut assets = OpAssets::default();
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
use std::io::{Read, Write};
use sha2::{Digest, Sha256};
use crate::ingress_meta::{self, DEFAULT_UA, FetchMeta};
let cfg = HttpFetchConfig::parse(with)?;
let pin = cfg.pinned_digest()?;
let out = ctx.dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
// The shared cache, unless this fetch carries a credential — see
// `is_credentialed`. Every use below is `Option`-guarded, so a run without one
// takes exactly the path this operator took before the cache existed.
let cache = ctx.cache.filter(|_| !cfg.is_credentialed());
let prior = ingress_meta::read(&out).filter(|_| out.exists());
// ── Pinned ───────────────────────────────────────────────────────────────
// A pinned digest names the bytes, and bytes that are already named have
// nothing to revalidate: the origin cannot tell this Protocol anything it did
// not already say it wanted. This is the ONLY route that skips the origin, and
// it is the reason the default route can afford not to.
if let Some(ref want) = pin {
if out.exists()
&& crate::fetch_cache::hash_file(&out).is_ok_and(|on_disk| on_disk == *want)
{
if let Some(c) = cache
&& !c.holds(&cfg.url)
{
let _ = c.store(&pinned_meta(&cfg, want, prior.as_ref()), &out);
}
return Ok(unchanged());
}
if let Some(c) = cache {
// The URL first, then the digest on its own: an object filed by
// another URL is still the artifact this manifest asked for, because
// the manifest asked for it by hash.
let entry = c.lookup(&cfg.url);
let hit = entry
.as_ref()
.is_some_and(|e| e.sha256.eq_ignore_ascii_case(want))
|| c.pinned_object(want).is_some();
if hit {
let meta = pinned_meta(&cfg, want, entry.as_ref());
return serve_from_cache(c, &cfg, &meta, &out);
}
}
}
// ── Revalidated ──────────────────────────────────────────────────────────
// What is known about these bytes already: the Protocol's own sidecar, which
// describes the copy in `out`, or — when it has none — the shared cache's
// entry for the same URL, verified on the way out of the store. Either way the
// request below carries a validator, so an unchanged remote answers `304` and
// the payload is not transferred a second time.
let shared = match (&prior, cache) {
(None, Some(c)) => c.lookup(&cfg.url),
_ => None,
};
let known = prior.as_ref().or(shared.as_ref());
// Redirects are followed here rather than by ureq. ureq follows them itself by
// default, and by the time it returns, the only headers left on the `Response`
// are the last hop's — for a resolve URL that redirects to a signed, expiring
// storage URL, that is the storage object's validator, which the origin does
// not accept in a later `If-None-Match`.
let agent = ureq::builder().redirects(0).build();
let mut url = cfg.url.clone();
let mut hop = 0u32;
let mut identity = RemoteIdentity::default();
let resp = loop {
// Default UA first, then Protocol overrides (a `User-Agent` key wins).
let mut req = agent.get(&url).set("User-Agent", DEFAULT_UA);
for (k, v) in &cfg.headers {
// ureq drops `Authorization` and `Cookie` when it follows a redirect;
// following by hand has to drop them too, or a Protocol's credential
// reaches whichever host the origin points at.
if hop > 0
&& (k.eq_ignore_ascii_case("authorization") || k.eq_ignore_ascii_case("cookie"))
{
continue;
}
req = req.set(k, v);
}
// The freshness contract: where this artifact has been fetched before —
// by this Protocol, or by any Protocol, into the shared cache — replay the
// stored ETag / Last-Modified as a conditional request. Sent on every hop,
// because the stored validator belongs to whichever hop offered one — for
// a redirect that carries no validator of its own that is the redirect
// target, which is only asked if the header travels with the fetch. An
// unchanged remote answers `304` and we keep the bytes.
if let Some(p) = known {
if let Some(ref etag) = p.etag {
req = req.set("If-None-Match", etag);
}
if let Some(ref lm) = p.last_modified {
req = req.set("If-Modified-Since", lm);
}
}
let resp = match req.call() {
Ok(resp) => resp,
// 304 Not Modified — the remote is byte-unchanged. The Protocol's own
// copy (and its sidecar) stay as they are, so its content identity —
// hence downstream staleness — is stable; a Protocol with no copy of
// its own takes the confirmed bytes from the shared cache.
Err(ureq::Error::Status(304, _)) => {
return keep_or_materialise(&cfg, &out, prior.as_ref(), shared.as_ref(), cache);
}
Err(e) => return Err(fetch_failed(format!("http_fetch: GET {}: {}", cfg.url, e))),
};
if resp.status() == 304 {
return keep_or_materialise(&cfg, &out, prior.as_ref(), shared.as_ref(), cache);
}
if hop == 0 {
identity = RemoteIdentity::read(&resp);
// The origin declared the artifact's content hash in the response head.
// Where it is the hash of bytes we already hold — in this Protocol or
// in the shared cache — the body is not read at all, and the
// declaration is recorded even if the sidecar predates the field, which
// costs no transfer.
//
// The comparison is against `sha256` and nothing else, because `sha256`
// is the one field of a record that has been hashed against the bytes
// it describes: `FetchCache::store` refuses bytes that do not match it
// and `FetchCache::lookup` re-hashes the object before returning it.
// `content_sha256` is the origin's word carried verbatim from a response
// head into the record, so a declaration matched against *it* is a
// record confirming itself — and the case it would decide is the one
// where the verified hash says these are other bytes.
if let Some(declared) = identity.content_sha256.as_deref()
&& let Some(p) = known
&& p.sha256 == declared
{
if prior.is_some() && p.content_sha256.as_deref() != Some(declared) {
let mut refreshed = p.clone();
refreshed.content_sha256 = Some(declared.to_string());
let _ = ingress_meta::write(&out, &refreshed);
}
return keep_or_materialise(&cfg, &out, prior.as_ref(), shared.as_ref(), cache);
}
}
let location = resp.header("Location").map(str::to_string);
match location {
Some(loc) if (300..400).contains(&resp.status()) => {
hop += 1;
if hop > MAX_REDIRECTS {
return Err(fetch_failed(format!(
"http_fetch: GET {}: more than {} redirects",
cfg.url, MAX_REDIRECTS
)));
}
url = join_location(&url, &loc).ok_or_else(|| {
fetch_failed(format!(
"http_fetch: GET {}: unresolvable redirect to {}",
cfg.url, loc
))
})?;
}
_ => break resp,
}
};
// An origin whose 3xx carries no `ETag`, `Last-Modified` or `X-Linked-ETag` —
// the shape of an http→https upgrade or a release redirect — leaves nothing to
// record, and a sidecar with no validator forfeits the next `304`. Take the
// final hop's instead; the conditional above is sent to every hop, so it
// reaches the hop that issued it.
if identity.offers_nothing() {
identity = RemoteIdentity::read(&resp);
}
// Stream to a sibling `.part` file, hashing as we go, then atomically rename —
// a killed run never leaves a truncated artifact that looks complete.
let tmp = out.with_extension("part");
let mut reader = resp.into_reader();
let mut file = std::fs::File::create(&tmp)
.map_err(|e| fetch_failed(format!("http_fetch: create {}: {}", tmp.display(), e)))?;
let mut hasher = Sha256::new();
let mut buf = [0u8; 65536];
loop {
let n = reader
.read(&mut buf)
.map_err(|e| fetch_failed(format!("http_fetch: read body {}: {}", cfg.url, e)))?;
if n == 0 {
break;
}
file.write_all(&buf[..n])
.map_err(|e| fetch_failed(format!("http_fetch: write {}: {}", tmp.display(), e)))?;
hasher.update(&buf[..n]);
}
let _ = file.sync_all();
drop(file);
let digest = format!("{:x}", hasher.finalize());
// A pin the transfer does not satisfy is refused before the bytes land: the
// manifest said which artifact it wanted and this is not it. Non-retryable —
// the same URL will serve the same bytes on the next attempt — and the partial
// file goes, so nothing downstream can read what was rejected.
if let Some(ref want) = pin
&& !digest.eq_ignore_ascii_case(want)
{
let _ = std::fs::remove_file(&tmp);
return Err(Error::StepExecution {
step: "http_fetch".to_string(),
source: std::io::Error::other(format!(
"{} served bytes hashing to {digest}, but the Protocol pinned {want}",
cfg.url
)),
});
}
std::fs::rename(&tmp, &out)
.map_err(|e| fetch_failed(format!("http_fetch: rename {}: {}", out.display(), e)))?;
// Persist the content identity for next run's conditional request + the
// `fresh` precondition's HEAD probe. Best-effort: a failed sidecar write
// doesn't fail the fetch (it just forfeits the next conditional/skip).
let meta = FetchMeta {
url: cfg.url.clone(),
request_headers: cfg.headers.clone(),
etag: identity.etag,
last_modified: identity.last_modified,
sha256: digest,
content_sha256: identity.content_sha256,
fetched_unix: now_unix(),
};
let _ = ingress_meta::write(&out, &meta);
// And file the bytes so the next Protocol to name this URL revalidates rather
// than transfers. Best-effort for the same reason: a cache that will not take
// a write costs a future transfer, not this one.
if let Some(c) = cache {
let _ = c.store(&meta, &out);
}
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// opendal_fetch (ingress) — the same fetch over **Apache OpenDAL**, a single
// operator that reads a scheme-dispatched `from` URL across 40+ backends
// (http/https, s3, gcs, azblob, …) with a built-in `RetryLayer` (per-request
// backoff, *on top of* the engine step-retry).
//
// MEASURED TRADE-OFF (2026-07-14 spike): OpenDAL's Http *builder* exposes only
// basic-auth / bearer — no header setter — but arbitrary request headers (a SEC
// User-Agent) ARE reachable by injecting a reqwest client (`default_headers`)
// through a custom `ReqwestTransport` (wired below; proven on the live SEC
// UA-gated fetch). The cost is +2 deps and ~25 lines of transport plumbing vs
// ureq's one `.set(k, v)`. OpenDAL also buffers the whole object in memory
// (blocking `read`) and needs a tokio runtime, where ureq streams with zero
// runtime. OpenDAL's genuine win is **s3://securelake** and multi-backend
// portability — one operator, many stores. Feature-gated behind `opendal`
// (Cargo.toml) so the default binary stays lean.
// ─────────────────────────────────────────────────────────────────────────────
#[cfg(feature = "opendal")]
struct OpendalFetch;
#[cfg(feature = "opendal")]
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct OpendalFetchConfig {
/// Scheme-dispatched source: `https://host/key`, `s3://bucket/key`, …
from: String,
/// Destination path, relative to the manifest directory. Written atomically.
to: String,
/// Extra request headers. Rejected on the http(s) backend (see above).
#[serde(default)]
headers: BTreeMap<String, String>,
}
#[cfg(feature = "opendal")]
impl OpendalFetchConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("opendal_fetch: invalid `with:` config: {}", e))
})
}
}
#[cfg(feature = "opendal")]
impl Operator for OpendalFetch {
fn name(&self) -> &'static str {
"opendal_fetch"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = OpendalFetchConfig::parse(with)?;
let mut assets = OpAssets::default();
assets.record_produces(cfg.to.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
use opendal::layers::RetryLayer;
use opendal::{Operator as DalOperator, services};
let cfg = OpendalFetchConfig::parse(with)?;
let out = ctx.dir.join(&cfg.to);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
let (scheme, rest) = cfg.from.split_once("://").ok_or_else(|| {
Error::ManifestValidation(format!(
"opendal_fetch: `from` must be a scheme URL (https://… or s3://…), got '{}'",
cfg.from
))
})?;
// authority = host or bucket; path = the object key relative to backend root.
let (authority, path) = rest.split_once('/').unwrap_or((rest, ""));
let op = match scheme {
"http" | "https" => {
let endpoint = format!("{}://{}", scheme, authority);
let builder = services::Http::default().endpoint(&endpoint);
let mut dal = DalOperator::new(builder)
.map_err(|e| fetch_failed(format!("opendal_fetch: http backend: {}", e)))?;
// OpenDAL's Http builder has no header setter (only basic-auth /
// bearer). To carry request headers — e.g. the SEC User-Agent —
// inject a reqwest client with `default_headers` through a custom
// ReqwestTransport. More plumbing (+2 deps) than ureq's `.set()`,
// but it works: proven on the live SEC fetch.
if !cfg.headers.is_empty() {
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
let mut hm = HeaderMap::new();
for (k, v) in &cfg.headers {
let name = HeaderName::from_bytes(k.as_bytes()).map_err(|e| {
Error::ManifestValidation(format!(
"opendal_fetch: bad header name '{}': {}",
k, e
))
})?;
let val = HeaderValue::from_str(v).map_err(|e| {
Error::ManifestValidation(format!(
"opendal_fetch: bad header value for '{}': {}",
k, e
))
})?;
hm.insert(name, val);
}
let client = reqwest::Client::builder()
.default_headers(hm)
.build()
.map_err(|e| {
fetch_failed(format!("opendal_fetch: build http client: {}", e))
})?;
let transport = opendal_http_transport_reqwest::ReqwestTransport::new(client);
dal = dal.with_context(
opendal::OperationContext::new()
.with_http_transport(opendal::HttpTransporter::new(transport)),
);
}
dal
}
"s3" => {
// Credentials/region from the standard AWS_* environment — the
// securelake path. Constructed + retry-wrapped here; exercised
// when securelake creds are present (not in this spike's run).
let builder = services::S3::default().bucket(authority);
DalOperator::new(builder)
.map_err(|e| fetch_failed(format!("opendal_fetch: s3 backend: {}", e)))?
}
other => {
return Err(Error::ManifestValidation(format!(
"opendal_fetch: unsupported scheme '{}' (http, https, s3)",
other
)));
}
};
let op = op.layer(RetryLayer::new());
// OpenDAL's blocking bridge requires an entered tokio runtime.
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.map_err(|e| fetch_failed(format!("opendal_fetch: tokio runtime: {}", e)))?;
let _guard = rt.enter();
let bop = opendal::blocking::Operator::new(op)
.map_err(|e| fetch_failed(format!("opendal_fetch: blocking bridge: {}", e)))?;
let buf = bop
.read(path)
.map_err(|e| fetch_failed(format!("opendal_fetch: read {}: {}", cfg.from, e)))?;
let tmp = out.with_extension("part");
std::fs::write(&tmp, buf.to_vec())
.map_err(|e| fetch_failed(format!("opendal_fetch: write {}: {}", tmp.display(), e)))?;
std::fs::rename(&tmp, &out)
.map_err(|e| fetch_failed(format!("opendal_fetch: rename {}: {}", out.display(), e)))?;
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// datapackage_describe (activate) — emit the Frictionless datapackage.json from the
// built Parquet: finetype types every column (the machine half) overlaid with the
// curated descriptor.overrides.json (the hand half; overrides win, relational keys
// hard-checked against the Parquet).
//
// NATIVE RUST — no Python, no `uv`. This operator used to wrap an arcform-embedded
// copy of describe.py via the uv-run substrate (see CHANGELOG); the JSON merge is
// now `merge_datapackage` below, over `serde_json::Value`. A Frictionless Data
// Package is a specification, not a runtime, and merging two JSON objects needs
// nothing Python provided — so the machine-decidable half still comes from
// shelling the `finetype` CLI (`finetype profile … -o datapackage`, must be on
// PATH, exactly as before) via the shared [`run_process`] substrate, and only the
// merge that used to run inside describe.py moved into this process. The typing
// boundary is untouched: finetype decides what it decides, this operator still
// forms no opinion about column types, it only plumbs the two halves together.
//
// Verified byte-identical to the retired uv/Python path against the real Parquet +
// descriptor.overrides.json for all four published datasets, save the one field
// that was ALREADY non-deterministic before this change: `created`, the per-run
// timestamp `finetype profile` stamps into its own output on every invocation.
// `x-finetype-version` — stamped by this operator, not by finetype — matches
// exactly, because both runs resolve and shell the same `finetype` on PATH.
//
// Version deliberately NOT bumped (`1.0.0`, unchanged): the `with:` contract and
// the produced bytes are identical, matching the precedent set when
// `x-finetype-version` was added (also not a version bump — see CHANGELOG). The
// reproducibility guarantee `materialize_frozen_script`'s write-if-changed cache
// gave the embedded script — `op@<version>` addresses exact bytes, so a behaviour
// change is a version bump and a rebuild, never a silent edit — still holds, by a
// different and simpler mechanism: there is no separate script materialized at
// runtime anymore, so the operator's behaviour is entirely the compiled binary,
// exactly like every other in-process operator in this catalog
// (`parquet_export`, `archive_extract`, `finetype_validate`) that never needed
// `materialize_frozen_script` to make that same claim.
// ─────────────────────────────────────────────────────────────────────────────
/// The floor finetype version whose taxonomy types these datasets correctly. A
/// drifted, older binary on PATH once silently shipped wrong labels — the guard
/// below turns that into a stopped run. Bump this when a newer finetype is needed
/// for correct labels; a Protocol may override it with `expect_finetype_version`.
///
/// 0.6.54 is the release that stopped eight-digit numbers typing as confident
/// dates. Up to and including 0.6.53 the year-first and day-first compact date
/// leaves both validated on `^\d{8}$`, so any eight-digit token — a financial
/// figure, a surrogate key — came back as a high-confidence date WITH A `strptime`
/// TRANSFORM ATTACHED. That is worse than a wrong label: a consumer that follows
/// the transform gets a corrupted column, not a mislabelled one. 0.6.53 therefore
/// has to be refused as firmly as 0.6.52 was.
const MIN_FINETYPE_VERSION: &str = "0.6.54";
/// The first finetype release whose `profile` subcommand takes `--nominations <FILE>`.
/// A step that sets `nominations:` is held to this floor instead of
/// `MIN_FINETYPE_VERSION`, because an older binary does not refuse the declaration
/// politely: clap rejects the unknown flag inside the subprocess, and the run dies on
/// a usage error that says nothing about which release to install. 0.6.59 has no such
/// flag and 0.6.60 is the tag that added it.
const MIN_FINETYPE_NOMINATIONS_VERSION: &str = "0.6.60";
/// Pull the first dotted numeric version out of `text` (e.g. `"finetype 0.6.53"`).
fn parse_finetype_version(text: &str) -> Option<(u64, u64, u64)> {
let re = regex::Regex::new(r"(\d+)\.(\d+)\.(\d+)").ok()?;
let caps = re.captures(text)?;
Some((
caps.get(1)?.as_str().parse().ok()?,
caps.get(2)?.as_str().parse().ok()?,
caps.get(3)?.as_str().parse().ok()?,
))
}
/// Fail closed unless the `finetype` on PATH is `>= min_version`.
///
/// Column typing is only as good as the installed binary, and a stale one is
/// invisible here — its output is still valid JSON. So assert the version before
/// profiling rather than trust whatever PATH resolves to.
///
/// Returns the resolved dotted version string (e.g. `"0.6.60"`) on success — the
/// SAME string this call already parsed to check the floor, reused by the caller
/// both to stamp `x-finetype-version` and to check `expect_finetype_version`.
/// Resolving the version once and threading it through means the stamp can only
/// ever name the binary this run actually asserted and ran, never a second,
/// independent lookup that could in principle disagree with the first.
///
/// `nominations` is whether this step passes `--nominations`. When it does, the
/// nominations floor is checked FIRST: it sits above `MIN_FINETYPE_VERSION`, so a
/// binary below both is refused with the reason that actually applies to this step
/// rather than with the general one.
fn require_finetype(min_version: &str, nominations: bool, ctx: &OpContext) -> Result<String> {
let out = run_process(
"finetype",
&["--version".to_string()],
ctx,
OutputMode::Capture,
"datapackage_describe",
)?;
let stdout = out.stdout.unwrap_or_default();
let got = resolve_finetype_version(&stdout)?;
if nominations {
check_nominations_floor(got, MIN_FINETYPE_NOMINATIONS_VERSION)?;
}
check_finetype_floor(got, min_version)?;
Ok(format!("{}.{}.{}", got.0, got.1, got.2))
}
/// Parse `finetype --version`'s stdout into the resolved dotted version, or the
/// refusal for output that does not contain one — PURE, no subprocess. Split out
/// of `require_finetype` so the unparseable-output refusal is testable without
/// spawning anything: a test pinning "output with no dotted triple is refused"
/// has no business depending on whether a fork/exec succeeds, and coupling it to
/// one made the test vulnerable to transient spawn flakiness under a loaded
/// machine that has nothing to do with the behaviour being pinned.
fn resolve_finetype_version(stdout: &str) -> Result<(u64, u64, u64)> {
parse_finetype_version(stdout).ok_or_else(|| {
fetch_failed(format!(
"datapackage_describe: could not parse a version from `finetype --version` ({:?}).",
stdout
))
})
}
/// Fail closed unless `got` is `>= min_version` — PURE, no subprocess. Split out of
/// `require_finetype` for the same reason `resolve_finetype_version` is: the floor
/// is a boundary condition (an EXACT match must pass), and a test for it should
/// exercise only the comparison, not a real `finetype` spawn.
fn check_finetype_floor(got: (u64, u64, u64), min_version: &str) -> Result<()> {
let want = parse_finetype_version(min_version)
.expect("MIN_FINETYPE_VERSION must itself parse as a dotted version");
if got < want {
return Err(fetch_failed(format!(
"datapackage_describe: finetype {}.{}.{} on PATH is older than the required {}; \
its labels are not trustworthy for these datasets. Update it \
(e.g. `cargo install --path crates/finetype-cli --force`) and re-run.",
got.0, got.1, got.2, min_version
)));
}
Ok(())
}
/// Fail closed unless `got` is `>= min_version`, for a step that declares
/// `nominations:` — PURE, no subprocess, and split from `check_finetype_floor` for
/// its message: that one says the labels are untrustworthy, and a binary that merely
/// predates `--nominations` is refused for a different reason, which is the one a
/// maintainer needs to read. The message names both versions.
fn check_nominations_floor(got: (u64, u64, u64), min_version: &str) -> Result<()> {
let want = parse_finetype_version(min_version)
.expect("MIN_FINETYPE_NOMINATIONS_VERSION must itself parse as a dotted version");
if got < want {
return Err(fetch_failed(format!(
"datapackage_describe: this step declares `nominations:`, which needs finetype \
{} or newer — the first release whose `profile` takes `--nominations` — but \
finetype {}.{}.{} on PATH is older and would reject the flag. Update it \
(e.g. `cargo install --path crates/finetype-cli --force`) or remove \
`nominations:` from the step.",
min_version, got.0, got.1, got.2
)));
}
Ok(())
}
/// Fail closed unless the already-resolved `resolved` version equals `expect` — the
/// manifest's `expect_finetype_version` pin.
///
/// `MIN_FINETYPE_VERSION` is a floor: anything newer passes, because a newer
/// finetype only ever has a taxonomy at least as good. `expect_finetype_version`
/// is a pin: a caller that wants THIS EXACT release run (reproducing a prior
/// descriptor, or holding a pipeline to a release under evaluation) asks for that
/// directly, since "newer is fine" is not always true for a pinned pipeline — the
/// engine that describes a dataset changing out from under it, silently, is
/// exactly the failure `x-finetype-version` exists to make visible, and refusing
/// up front is cheaper than discovering it by reading the stamp after the fact.
fn require_exact_finetype_version(resolved: &str, expect: &str) -> Result<()> {
if resolved != expect {
return Err(fetch_failed(format!(
"datapackage_describe: finetype on PATH reports {}, but the manifest's \
`expect_finetype_version` requested exactly {}. Resolve which binary should \
produce this descriptor before running (e.g. `cargo install --path \
crates/finetype-cli --force` for a specific tag, or adjust \
`expect_finetype_version` if a newer release is intended).",
resolved, expect
)));
}
Ok(())
}
/// Run `finetype profile -f <parquet> -o datapackage` and parse its stdout as the
/// Frictionless Data Package finetype computed from the built Parquet directly
/// (including the resource `bytes` / `hash` / `format` / `mediatype`).
///
/// `nominations`, when present, is appended as `--nominations <FILE>`, resolved
/// against the manifest directory by the caller exactly as `parquet` is, so the argv
/// names one file on its own rather than one that depends on `run_process` also
/// setting the child's working directory, and so an error finetype prints about the
/// file names the path that was actually opened.
fn finetype_datapackage(
parquet: &Path,
nominations: Option<&Path>,
ctx: &OpContext,
) -> Result<serde_json::Value> {
let mut args = vec![
"profile".to_string(),
"-f".to_string(),
parquet.display().to_string(),
"-o".to_string(),
"datapackage".to_string(),
];
if let Some(nominations) = nominations {
args.push("--nominations".to_string());
args.push(nominations.display().to_string());
}
let out = run_process(
"finetype",
&args,
ctx,
OutputMode::Capture,
"datapackage_describe",
)?;
let stdout = out.stdout.unwrap_or_default();
serde_json::from_str(&stdout).map_err(|e| {
fetch_failed(format!(
"datapackage_describe: could not parse finetype output as JSON: {}",
e
))
})
}
/// Top-level override keys handled structurally — everything else is copied to the
/// package root as-is (title / description / homepage / licenses / sources / …).
const MERGE_STRUCTURAL: &[&str] = &["resource", "fields", "primaryKey", "foreignKeys"];
/// The per-field key finetype stamps on a column whose type was DECLARED in
/// `nominations.finetype.json` rather than inferred from the data. A sidecar that
/// could set it would make a hand-typed field indistinguishable from a declared one,
/// which is the exact distinction the mark exists to carry.
///
/// [`check_field_claims`] refuses it in a curated `fields` block, which is the block
/// the four published sidecars use. It does NOT see a `resource.schema` or a top-level
/// `resources` override, both of which replace the field array wholesale and can carry
/// the mark through — see the CHANGELOG entry, which names that hole rather than
/// leaving a reader to infer from this guard that the whole class is closed.
const NOMINATED_KEY: &str = "x-finetype-nominated";
/// The per-field keys that state a typing verdict: the type itself, the taxonomy
/// label behind it and the engine's confidence in it. On a nominated field these are
/// the nomination's to set; on any field the sidecar setting one is pre-empting
/// finetype's typing rather than curating around it.
const NOMINATION_CLAIM_KEYS: &[&str] = &["type", "x-finetype-label", "x-finetype-confidence"];
/// Where a declared type belongs instead of the curated sidecar. Named in every
/// refusal and every warning below, because "this is wrong" without "put it here"
/// leaves the maintainer to guess.
const NOMINATIONS_FILE: &str = "nominations.finetype.json";
/// Overlay the curated `descriptor.overrides.json` sidecar onto finetype's base
/// descriptor — overrides win, finetype supplies everything the sidecar does not
/// mention. Mirrors the retired describe.py's `_merge` + `_check_relations` for the
/// four structural keys, the per-field shallow-replace semantics (an override
/// supplies a key, finetype's value for every OTHER key on that field survives) and
/// the primaryKey/foreignKey drift hard-fail, and adds one thing describe.py never
/// had: [`check_field_claims`], which refuses a sidecar that forges or contradicts a
/// finetype NOMINATION before any of that copying happens.
fn merge_datapackage(
mut base: serde_json::Value,
overrides: &serde_json::Value,
) -> Result<serde_json::Value> {
let overrides_obj = overrides.as_object().ok_or_else(|| {
fetch_failed(
"datapackage_describe: descriptor.overrides.json must be a JSON object".to_string(),
)
})?;
// 1) Package-level curated keys (title, description, homepage, licenses, sources, …).
{
let base_obj = base.as_object_mut().ok_or_else(|| {
fetch_failed(
"datapackage_describe: finetype's profile output must be a JSON object".to_string(),
)
})?;
for (key, value) in overrides_obj {
if !MERGE_STRUCTURAL.contains(&key.as_str()) {
base_obj.insert(key.clone(), value.clone());
}
}
}
// 2) Resource-level overrides (e.g. the published `path`; finetype writes the
// local build path, which is not where consumers fetch the resource).
let resource_obj = base
.get_mut("resources")
.and_then(|r| r.as_array_mut())
.and_then(|arr| arr.first_mut())
.and_then(|r| r.as_object_mut())
.ok_or_else(|| {
fetch_failed("datapackage_describe: finetype's output has no resources[0]".to_string())
})?;
if let Some(res_overrides) = overrides_obj.get("resource").and_then(|v| v.as_object()) {
for (key, value) in res_overrides {
resource_obj.insert(key.clone(), value.clone());
}
}
let schema = resource_obj
.get_mut("schema")
.and_then(|s| s.as_object_mut())
.ok_or_else(|| fetch_failed("datapackage_describe: resource has no schema".to_string()))?;
let fields = schema
.get_mut("fields")
.and_then(|f| f.as_array_mut())
.ok_or_else(|| fetch_failed("datapackage_describe: schema has no fields".to_string()))?;
// The columns the built Parquet actually has, per finetype — computed BEFORE
// the per-field override loop below mutates field contents (it never touches
// `name`), so this set stays valid for both the stale-override warning and the
// relational drift check after it.
let present: std::collections::BTreeSet<String> = fields
.iter()
.filter_map(|f| f.get("name").and_then(|n| n.as_str()).map(str::to_string))
.collect();
// The columns finetype marked NOMINATED, each with the `constraints` object the
// engine carried for it. Captured here for the same reason as `present` and one
// more: the per-field loop below overwrites both the mark and the constraints, so
// a guard reading them afterwards would be comparing the sidecar's claim against
// the sidecar's own value. It has to compare against what the ENGINE declared.
let nominated: std::collections::BTreeMap<String, serde_json::Value> = fields
.iter()
.filter(|f| f.get(NOMINATED_KEY).and_then(|v| v.as_bool()) == Some(true))
.filter_map(|f| {
let name = f.get("name").and_then(|n| n.as_str())?.to_string();
let constraints = f
.get("constraints")
.cloned()
.unwrap_or(serde_json::Value::Null);
Some((name, constraints))
})
.collect();
// 3) Per-field overrides, matched by column name (adds `description`, or
// corrects a `x-finetype-label`, etc.). A sidecar field that does not exist
// in the Parquet is a stale override — warn so it gets cleaned up.
if let Some(field_overrides) = overrides_obj.get("fields").and_then(|v| v.as_object()) {
// Before any copying: a sidecar may curate around finetype's typing, never
// over it. Refuses on a forged or contradicted nomination; warns on a claim
// that merely pre-empts one.
for line in check_field_claims(field_overrides, &nominated)? {
eprintln!("{line}");
}
for name in field_overrides.keys() {
if !present.contains(name) {
eprintln!(
"datapackage_describe: WARNING field override '{name}' is not in the \
Parquet (stale sidecar entry?)"
);
}
}
for field in fields.iter_mut() {
let name = field
.get("name")
.and_then(|n| n.as_str())
.map(str::to_string);
let Some(name) = name else { continue };
if let Some(field_over) = field_overrides.get(&name).and_then(|v| v.as_object()) {
let field_obj = field
.as_object_mut()
.expect("finetype's field entries are JSON objects");
for (key, value) in field_over {
field_obj.insert(key.clone(), value.clone());
}
}
}
}
// 4) Relational metadata. These are hand-curated and cannot be inferred, but
// they MUST reference real columns — hard-fail on drift.
for rel_key in ["primaryKey", "foreignKeys"] {
if let Some(value) = overrides_obj.get(rel_key) {
schema.insert(rel_key.to_string(), value.clone());
}
}
check_relations(schema, &present)?;
Ok(base)
}
/// Decide, for every field block in the curated sidecar, whether it is curating
/// around finetype's typing or over it.
///
/// Three outcomes partition the input space, and the refusal outranks both warnings
/// wherever their givens also match:
///
/// * **Refuse — forgery.** The block sets `x-finetype-nominated` itself. That mark is
/// finetype's statement that a human declared this column's type in
/// `nominations.finetype.json`; a sidecar that can write it can make a hand-typed
/// field claim to be a declared one, and the mark stops meaning anything.
/// * **Refuse — contradiction.** The block sets `type`, `x-finetype-label` or
/// `x-finetype-confidence` on a column finetype already marked nominated. A
/// nomination is taken as given, so nothing downstream may overturn it — least of
/// all a file with no contract on the far side of the pipeline.
/// * **Warn — pre-emption.** The same three keys on a column nobody nominated. Nine
/// live fields across three published sidecars do exactly this today, and open-analytics
/// CI never runs `arc`, so a refusal here would not redden that repo's own gates — it
/// would fail the next rebuild of three published datasets instead. The warning is the
/// deprecation notice; the refusal follows when the last sidecar is clean.
/// * **Warn — lost bounds.** A nominated column whose `constraints` the sidecar
/// replaces, and which claims none of the three keys above. Tightening bounds under a
/// nominated label is legitimate curation, so this is not a refusal; losing the
/// nominated bounds without noticing is what the warning exists to stop, which is why
/// it prints both sets.
///
/// `nominated` maps each column finetype marked nominated to the `constraints` the
/// ENGINE carried for it, read before the merge overwrote anything.
///
/// Warnings are RETURNED rather than printed, for two reasons. A refusal aborts the
/// step, so advice about a field in a descriptor that will never be written is noise
/// and is discarded. And a returned value can be compared exactly by a test, while a
/// side effect can only be searched: every assertion on this function's output was
/// once `stderr.contains(<substring>)`, which is monotone — adding a line or swapping
/// one rendering for another leaves every substring present — so no branch whose only
/// effect is on message CONTENT could be pinned at all. Three such branches shipped
/// unpinned and the mutation gate caught all three.
fn check_field_claims(
field_overrides: &serde_json::Map<String, serde_json::Value>,
nominated: &std::collections::BTreeMap<String, serde_json::Value>,
) -> Result<Vec<String>> {
let mut forged: Vec<String> = Vec::new();
let mut contradicted: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
for (name, block) in field_overrides {
let Some(block) = block.as_object() else {
continue;
};
let base_constraints = nominated.get(name);
let claims: Vec<&str> = NOMINATION_CLAIM_KEYS
.iter()
.copied()
.filter(|key| block.contains_key(*key))
.collect();
if block.contains_key(NOMINATED_KEY) {
// Checked before the contradiction arm and before both warnings, so a
// block that forges the mark AND claims a label is refused for the
// forgery rather than warned about for the label.
forged.push(format!("'{name}' → '{NOMINATED_KEY}'"));
continue;
}
let Some(base_constraints) = base_constraints else {
for key in &claims {
warnings.push(format!(
"datapackage_describe: WARNING field '{name}' sets '{key}' in \
descriptor.overrides.json, pre-empting finetype's typing. Declare \
the type in {NOMINATIONS_FILE} instead — this override will be \
refused once the published sidecars are clean."
));
}
continue;
};
if !claims.is_empty() {
for key in &claims {
contradicted.push(format!("'{name}' → '{key}'"));
}
continue;
}
if let Some(sidecar_constraints) = block.get("constraints") {
warnings.push(format!(
"datapackage_describe: WARNING field '{name}' is nominated and \
descriptor.overrides.json replaces its constraints — the bounds that \
came with the nomination in {NOMINATIONS_FILE} are gone unless the \
sidecar restates them.\n nominated: {}\n sidecar: {}",
render_constraints(base_constraints),
render_constraints(sidecar_constraints),
));
}
}
if forged.is_empty() && contradicted.is_empty() {
return Ok(warnings);
}
let mut detail = Vec::new();
if !forged.is_empty() {
detail.push(format!(
" forges the nomination mark: {}",
forged.join(", ")
));
}
if !contradicted.is_empty() {
detail.push(format!(
" contradicts a field finetype already nominated: {}",
contradicted.join(", ")
));
}
Err(fetch_failed(format!(
"datapackage_describe: descriptor.overrides.json states a type claim that is \
not its to state — a declared type, its label and its confidence belong in \
{NOMINATIONS_FILE}, which finetype reads, not in the curated sidecar, which \
nothing reads back.\n{}",
detail.join("\n")
)))
}
/// Render a field's `constraints` for a warning. A field finetype gave no constraints
/// at all reads as `(none)` rather than as the bare `null` a JSON render would produce,
/// because `null` looks like a value that was set.
fn render_constraints(value: &serde_json::Value) -> String {
if value.is_null() {
"(none)".to_string()
} else {
value.to_string()
}
}
/// Fail if a curated `primaryKey` / `foreignKeys` names a column absent from the
/// built Parquet — exactly the descriptor-drift this step exists to catch, and a
/// silent mismatch would be worse than a stopped run.
fn check_relations(
schema: &serde_json::Map<String, serde_json::Value>,
present: &std::collections::BTreeSet<String>,
) -> Result<()> {
let mut missing = Vec::new();
if let Some(pk) = schema.get("primaryKey").and_then(|v| v.as_array()) {
for col in pk.iter().filter_map(|v| v.as_str()) {
if !present.contains(col) {
missing.push(format!("primaryKey → '{col}'"));
}
}
}
if let Some(fks) = schema.get("foreignKeys").and_then(|v| v.as_array()) {
for fk in fks {
if let Some(cols) = fk.get("fields").and_then(|v| v.as_array()) {
for col in cols.iter().filter_map(|v| v.as_str()) {
if !present.contains(col) {
missing.push(format!("foreignKey → '{col}'"));
}
}
}
}
}
if missing.is_empty() {
return Ok(());
}
let cols = present.iter().cloned().collect::<Vec<_>>().join(", ");
Err(fetch_failed(format!(
"datapackage_describe: curated relational metadata references columns not in the \
built Parquet — the descriptor has drifted from package.sql. \
Reconcile descriptor.overrides.json.\n missing: {}\n columns: {}",
missing.join("; "),
cols
)))
}
struct DatapackageDescribe;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct DatapackageDescribeConfig {
/// Built Parquet finetype types (a `reads` asset — keeps `describe` downstream
/// of the step that produces it). Resolved against ctx.dir.
parquet: String,
/// Curated descriptor sidecar (JSON) overlaid onto finetype's base.
overrides: String,
/// datapackage.json to write (the `produces` asset). Resolved against ctx.dir.
out: String,
/// Pin the run to one exact finetype release. When set, the step refuses —
/// naming both versions — unless the resolved `finetype` on PATH reports
/// exactly this version, so a pinned pipeline cannot silently describe with a
/// different engine than the one it asked for. Independent of this operator's
/// own `MIN_FINETYPE_VERSION` floor (unconfigurable here — it covers taxonomy
/// correctness for every caller), which stays a "no older than" check even
/// when this is also set.
#[serde(default)]
expect_finetype_version: Option<String>,
/// Declared column types, passed to `finetype profile --nominations`. Resolved
/// against ctx.dir, and a `reads` asset, so editing the file re-describes on the
/// next warm run. Setting it raises the finetype floor to
/// `MIN_FINETYPE_NOMINATIONS_VERSION`.
#[serde(default)]
nominations: Option<String>,
}
impl DatapackageDescribeConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!(
"datapackage_describe: invalid `with:` config: {}",
e
))
})
}
}
impl Operator for DatapackageDescribe {
fn name(&self) -> &'static str {
"datapackage_describe"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 1, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = DatapackageDescribeConfig::parse(with)?;
let mut assets = OpAssets::default();
assets.record_reads(cfg.parquet.clone(), crate::asset_kind::AssetKind::File);
// Both curated inputs are reads. Neither is produced by any step, so without
// this the step's hash covered the Parquet and its own config and nothing
// else: an edit to either file left `describe` hash-clean, and the written
// descriptor kept describing the file as it was before the edit.
assets.record_reads(cfg.overrides.clone(), crate::asset_kind::AssetKind::File);
if let Some(nominations) = &cfg.nominations {
assets.record_reads(nominations.clone(), crate::asset_kind::AssetKind::File);
}
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = DatapackageDescribeConfig::parse(with)?;
let parquet = ctx.dir.join(&cfg.parquet);
let overrides_path = ctx.dir.join(&cfg.overrides);
let out = ctx.dir.join(&cfg.out);
let nominations = cfg.nominations.as_deref().map(|n| ctx.dir.join(n));
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
let version = require_finetype(MIN_FINETYPE_VERSION, nominations.is_some(), ctx)?;
if let Some(expect) = cfg.expect_finetype_version.as_deref() {
require_exact_finetype_version(&version, expect)?;
}
let base = finetype_datapackage(&parquet, nominations.as_deref(), ctx)?;
let overrides_text = std::fs::read_to_string(&overrides_path).map_err(|e| {
fetch_failed(format!(
"datapackage_describe: read {}: {}",
overrides_path.display(),
e
))
})?;
let overrides: serde_json::Value = serde_json::from_str(&overrides_text).map_err(|e| {
fetch_failed(format!(
"datapackage_describe: parse {}: {}",
overrides_path.display(),
e
))
})?;
let mut descriptor = merge_datapackage(base, &overrides)?;
// Stamped AFTER the merge, so a sidecar override cannot supply (or clobber)
// the identity of the engine that actually ran — this field is machine-derived
// like the rest of finetype's half of the descriptor, not hand-curated, and it
// is what makes a descriptor produced by a superseded finetype distinguishable
// from a current one just by reading the file.
descriptor
.as_object_mut()
.expect("merge_datapackage always returns a JSON object")
.insert(
"x-finetype-version".to_string(),
serde_json::Value::String(version),
);
let mut bytes = serde_json::to_vec_pretty(&descriptor).map_err(|e| {
fetch_failed(format!(
"datapackage_describe: serialize {}: {}",
out.display(),
e
))
})?;
bytes.push(b'\n');
std::fs::write(&out, &bytes).map_err(|e| {
fetch_failed(format!(
"datapackage_describe: write {}: {}",
out.display(),
e
))
})?;
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// finetype_validate (gate) — the fail-closed DRIFT GATE that makes finetype's typed
// schema LOAD-BEARING, not just post-hoc descriptive. Where datapackage_describe
// *describes* the built Parquet, this *enforces* it: it checks every column of the
// terminal Parquet against a self-derived JSON-Schema contract.
//
// IN-ENGINE (no subprocess). The gate opens a throwaway in-memory DuckDB, LOADs the
// finetype DuckDB extension, and runs its `ft_validate(table, schema) → TABLE` macro —
// the model-free (jsonschema-based, no model dir) validator. `ft_validate` melts every
// column to VARCHAR and checks each against its `$.properties.<col>` subschema,
// returning a per-column report {column_name, total, rejects, sample_message}. Any
// rejecting row → the run fails closed, and the offending columns are logged first.
// Running the extension in-process (vs shelling a `finetype` CLI) means the gate needs
// no `finetype`/`duckdb` on PATH — only the built extension artifact, located via the
// `FINETYPE_DUCKDB_EXT` env var (the deployment sets the per-platform build) or a
// per-step `extension:` override.
//
// The contract per dataset is derived from that dataset's OWN produced Parquet, relaxed
// to the string-surviving keywords `ft_validate` enforces once columns are melted to
// VARCHAR — `enum` (closed categorical domains), `pattern`, `minLength`/`maxLength`,
// `const`, `type: string`. Because it is derived from the data, every current row
// validates: the gate is a PASS-THROUGH today and a TRIPWIRE for future Runs (a value
// outside a closed enum, a length past the envelope, a shape break → rejects → the run
// fails). See each dataset's schema.finetype.json `$comment` for what is enforced (and
// what is deliberately NOT: numeric ranges are inert once melted, and NULLs are
// invisible — no not-null enforcement).
//
// CHECK-ONLY is load-bearing for byte-equivalence: the gate READS the Parquet via
// `read_parquet` and never rewrites/reorders/drops rows, so the published bytes are
// unchanged. `allow_unsigned_extensions` MUST be an open-time flag (a runtime `SET`
// fails), so the connection is opened with it via `open_in_memory_with_flags`.
// ─────────────────────────────────────────────────────────────────────────────
struct FinetypeValidate;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct FinetypeValidateConfig {
/// Built Parquet to check (a `reads` asset — orders the gate downstream of the
/// step that produces it, so a rebuilt Parquet re-triggers the gate). Resolved
/// against ctx.dir.
parquet: String,
/// Self-derived JSON-Schema contract to check against (a `reads` asset). Resolved
/// against ctx.dir.
schema: String,
/// OPTIONAL per-step override for the finetype DuckDB extension path. Normally the
/// path comes from the `FINETYPE_DUCKDB_EXT` env var (the deployment sets the
/// per-platform build); this field lets a single Protocol pin a specific artifact.
/// NOT a data asset — deliberately excluded from `assets()`, so manifests that omit
/// it (all of them) don't churn and lineage is unchanged.
#[serde(default)]
extension: Option<String>,
}
impl FinetypeValidateConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("finetype_validate: invalid `with:` config: {}", e))
})
}
}
impl Operator for FinetypeValidate {
fn name(&self) -> &'static str {
"finetype_validate"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = FinetypeValidateConfig::parse(with)?;
// A gate produces no artifact (check-only). It READS both the Parquet — which
// orders it downstream of the export that produces it (so a rebuilt Parquet
// re-triggers the gate via stale-propagation) — and the schema contract.
let mut assets = OpAssets::default();
assets.record_reads(cfg.parquet.clone(), crate::asset_kind::AssetKind::File);
assets.record_reads(cfg.schema.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = FinetypeValidateConfig::parse(with)?;
let parquet = ctx.dir.join(&cfg.parquet);
let schema = ctx.dir.join(&cfg.schema);
// Locate the finetype DuckDB extension (per-step override, else env var).
// A missing/unset artifact is a deterministic misconfiguration → non-retryable.
let ext = finetype_ext_path(cfg.extension.as_deref())?;
// Open a throwaway in-memory DuckDB with unsigned extensions ALLOWED. This is an
// OPEN-TIME flag — a runtime `SET allow_unsigned_extensions` is rejected — so it
// goes through `open_in_memory_with_flags`.
let config = duckdb::Config::default()
.allow_unsigned_extensions()
.map_err(|e| gate_failed(format!("configure duckdb: {e}")))?;
let conn = duckdb::Connection::open_in_memory_with_flags(config)
.map_err(|e| gate_failed(format!("open in-memory duckdb: {e}")))?;
conn.execute_batch(&format!("LOAD '{}';", sql_lit(&ext.display().to_string())))
.map_err(|e| gate_failed(format!("LOAD finetype extension {}: {e}", ext.display())))?;
// View the terminal Parquet — read-only, so the published bytes are untouched.
conn.execute_batch(&format!(
"CREATE OR REPLACE TEMP VIEW _ftv AS SELECT * FROM read_parquet('{}');",
sql_lit(&parquet.display().to_string())
))
.map_err(|e| gate_failed(format!("read_parquet {}: {e}", parquet.display())))?;
// Run the model-free `ft_validate` macro and pull the full per-column report.
// It auto-detects the `schema` arg as a file path (it does not start with `{`).
// coalesce guards the nested-column skip rows, which carry NULL total/rejects.
let report_sql = format!(
"SELECT column_name, coalesce(total, 0)::BIGINT, coalesce(rejects, 0)::BIGINT, \
coalesce(sample_message, '') FROM ft_validate('_ftv', '{}') \
ORDER BY coalesce(rejects, 0) DESC, column_name",
sql_lit(&schema.display().to_string())
);
let mut stmt = conn
.prepare(&report_sql)
.map_err(|e| gate_failed(format!("prepare ft_validate: {e}")))?;
let report = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?, // column_name
row.get::<_, i64>(1)?, // total
row.get::<_, i64>(2)?, // rejects
row.get::<_, String>(3)?, // sample_message
))
})
.map_err(|e| gate_failed(format!("run ft_validate: {e}")))?;
let mut total_rejects: i64 = 0;
let mut offenders: Vec<(String, i64, i64, String)> = Vec::new();
for row in report {
let (col, total, rejects, msg) =
row.map_err(|e| gate_failed(format!("read ft_validate report: {e}")))?;
total_rejects += rejects;
if rejects > 0 {
offenders.push((col, total, rejects, msg));
}
}
if total_rejects > 0 {
// The refusal carries the subject, the clause and an example — everything
// the reader's next two questions ask for ("which column?", "what value?"),
// all of it already in hand at the moment the gate decides to refuse. It
// goes in `StepFailed.stderr` and NOWHERE else: that field is what
// `Error`'s Display prints and therefore the only channel that survives
// anything wrapping `arc`. A second copy on `eprintln!` used to carry the
// per-column line and is deliberately gone — one failure, one report.
let mut detail = format!(
"finetype_validate: {} column(s) drifted from {} ({} rejecting row(s))",
offenders.len(),
cfg.schema,
total_rejects
);
for (col, total, rejects, msg) in &offenders {
detail.push_str(&format!(
"\n ✗ {col}: {rejects}/{total} row(s) reject — {}",
elide(msg, DRIFT_MESSAGE_WIDTH)
));
match sample_offending_values(&conn, &schema, col, DRIFT_SAMPLE_LIMIT) {
Ok(sample) => detail.push_str(&format!("\n{}", sample.render())),
// A sampling query that fails must not turn an actionable drift
// refusal into a confusing one: the column, the counts and
// ft_validate's own message above are already reported, so say
// what could not be sampled and keep going.
Err(e) => detail.push_str(&format!(
"\n offending values unavailable for {col}: {e}"
)),
}
}
return Err(Error::StepFailed {
step: String::new(), // runner rewrites with the step name
code: 1,
stderr: detail,
});
}
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
/// How many DISTINCT offending values a drift refusal reports per column. Bounded on
/// purpose: a column whose every row rejects would otherwise print the whole column
/// into a step error. The number is printed in the message, and the message says the
/// list was cut when it was, so the reader is never left guessing whether five was all
/// of them.
const DRIFT_SAMPLE_LIMIT: usize = 5;
/// How wide a single offending value is allowed to print before it is elided. A cell
/// can hold a whole document; the point is to recognise the value, not to transport it.
const DRIFT_VALUE_WIDTH: usize = 120;
/// How wide `ft_validate`'s own sample message may print. It quotes the offending value
/// back inside a sentence, so it is exactly as unbounded as the cell is — bounding the
/// sampled values and not this one would leave the whole document in the refusal after
/// all. Wider than [`DRIFT_VALUE_WIDTH`] because the value arrives wrapped in prose.
const DRIFT_MESSAGE_WIDTH: usize = 200;
/// The bounded evidence behind one drifted column: up to [`DRIFT_SAMPLE_LIMIT`] distinct
/// offending values, the clause each failed, how many rows carry it, and how many
/// distinct offenders there were in total (so the message can say the list was cut).
struct DriftSample {
/// (value, constraint token, rows carrying it), most-frequent first.
values: Vec<(String, String, i64)>,
/// Distinct (value, clause) pairs that reject — `values.len()` before the bound.
distinct_total: i64,
/// The bound that was applied, so the rendered line can state it.
limit: usize,
}
impl DriftSample {
fn render(&self) -> String {
if self.values.is_empty() {
// ft_validate counted rejects for this column but the per-cell re-query
// found none — a real disagreement, worth saying rather than printing an
// empty list that reads like "no values".
return " offending values: none re-derived (ft_validate and the \
per-cell check disagree)"
.to_string();
}
let cut = if (self.values.len() as i64) < self.distinct_total {
format!(" — list cut at the {} shown", self.limit)
} else {
String::new()
};
let rendered: Vec<String> = self
.values
.iter()
.map(|(v, clause, n)| format!("'{}' ×{n} [{clause}]", elide(v, DRIFT_VALUE_WIDTH)))
.collect();
format!(
" offending values, {} of {} distinct{}: {}",
self.values.len(),
self.distinct_total,
cut,
rendered.join(", ")
)
}
}
/// Truncate a value for display on a character boundary, marking that it was cut.
fn elide(s: &str, width: usize) -> String {
if s.chars().count() <= width {
return s.to_string();
}
let head: String = s.chars().take(width).collect();
format!("{head}…")
}
/// Re-derive the actual offending values for one drifted column.
///
/// `ft_validate` reports per-column counts and ONE sample message; it does not report
/// which values failed. `ft_validate_text` — the per-cell engine that same macro calls
/// under the hood — does, so this runs it over the column against the column's own
/// `$.properties.<col>` subschema and groups the rejects. `LIMIT` bounds what is
/// returned; the unbounded `count(*)` beside it is what makes "cut at 5 of 37" sayable.
///
/// Errors are returned as a string rather than an [`Error`]: the caller is already
/// building a refusal and must not have it replaced by a failure to decorate it.
fn sample_offending_values(
conn: &duckdb::Connection,
schema: &std::path::Path,
column: &str,
limit: usize,
) -> std::result::Result<DriftSample, String> {
// The column name lands in two places with different escaping rules: a quoted
// identifier (`"` doubled) and a JSON path segment inside a SQL string literal
// (`'` doubled). The macro this mirrors quotes the same way.
let ident = column.replace('"', "\"\"");
let path = format!("$.\"properties\".\"{}\"", column.replace('"', "\\\""));
let sql = format!(
"WITH sub(j) AS (\
SELECT json_extract((SELECT content FROM read_text('{schema}')), '{path}')::VARCHAR\
), bad AS (\
SELECT \"{ident}\"::VARCHAR AS v, \
ft_validate_text(\"{ident}\"::VARCHAR, (SELECT j FROM sub)) AS r FROM _ftv\
), d AS (\
SELECT v, r['constraint'] AS clause, count(*)::BIGINT AS n \
FROM bad WHERE NOT r['valid'] GROUP BY v, r['constraint']\
) \
SELECT v, clause, n, (SELECT count(*) FROM d)::BIGINT \
FROM d ORDER BY n DESC, v LIMIT {limit}",
schema = sql_lit(&schema.display().to_string()),
path = sql_lit(&path),
);
let mut stmt = conn
.prepare(&sql)
.map_err(|e| format!("prepare value sample: {e}"))?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, Option<String>>(0)?.unwrap_or_default(),
row.get::<_, Option<String>>(1)?
.unwrap_or_else(|| "?".into()),
row.get::<_, i64>(2)?,
row.get::<_, i64>(3)?,
))
})
.map_err(|e| format!("run value sample: {e}"))?;
let mut values = Vec::new();
let mut distinct_total = 0i64;
for row in rows {
let (v, clause, n, total) = row.map_err(|e| format!("read value sample: {e}"))?;
distinct_total = total;
values.push((v, clause, n));
}
Ok(DriftSample {
values,
distinct_total,
limit,
})
}
/// Resolve the finetype DuckDB extension path for the in-engine gate: a per-step
/// `extension:` override wins, else the `FINETYPE_DUCKDB_EXT` env var. A missing/unset
/// path is a deterministic misconfiguration → NON-retryable [`Error::StepExecution`]
/// (a bad binary must never burn a 4 h job's retries — parity with the old missing
/// `finetype` CLI case).
fn finetype_ext_path(config_ext: Option<&str>) -> Result<PathBuf> {
let raw = match config_ext {
Some(p) if !p.trim().is_empty() => p.to_string(),
_ => std::env::var("FINETYPE_DUCKDB_EXT").map_err(|_| Error::StepExecution {
step: "finetype_validate".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
"finetype DuckDB extension not configured: set FINETYPE_DUCKDB_EXT to the built \
finetype.duckdb_extension (per-platform), or pass `extension:` in the step",
),
})?,
};
let path = PathBuf::from(raw);
if !path.exists() {
return Err(Error::StepExecution {
step: "finetype_validate".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("finetype DuckDB extension not found at {}", path.display()),
),
});
}
Ok(path)
}
/// Map an in-engine gate failure (duckdb open / LOAD / query) to the retryable
/// [`Error::StepFailed`], matching the old shell-out's "finetype error exit → retryable"
/// posture (the runner rewrites the empty `step` with the step name).
fn gate_failed(msg: String) -> Error {
Error::StepFailed {
step: String::new(),
code: 1,
stderr: format!("finetype_validate: {msg}"),
}
}
/// Escape single quotes for a SQL string literal — used when interpolating filesystem
/// paths into `LOAD`/`read_parquet`/`ft_validate`.
fn sql_lit(s: &str) -> String {
s.replace('\'', "''")
}
// ─────────────────────────────────────────────────────────────────────────────
// html_link_discover (ingress) — the "list what's there" primitive: GET an index
// page, regex-pluck the hrefs matching `pattern`, absolutise them against the page
// URL, de-dup preserving document order, and emit a newline-delimited URL list as
// a first-class produced asset.
//
// This is the discovery half of the SEC N-CEN pull (scripts/fetch_ncen.sh): the
// SEC "Form N-CEN Data Sets" page lists each quarter's zip as a root-relative href
// (`/files/dera/data/…ncen_2024q1.zip`); this op turns that page into the list of
// absolute zip URLs a downstream fetch fans out over. Same default UA as http_fetch
// (SEC 403s a missing UA); a Protocol `User-Agent` in `headers` overrides it.
//
// NOTE (byte-equivalence): this + archive_extract are unit-verified building blocks;
// they do NOT by themselves retire fetch_ncen.sh end-to-end, because the engine has
// no fan-out step that maps over the discovered-quarters URL list yet. That fan-out
// is separate work; here we produce the complete list.
// ─────────────────────────────────────────────────────────────────────────────
// Gated with `http_fetch` (both are ureq-backed ingress ops); see the reachability
// note beside `http_fetch` above. The pure URL helpers below (`url_origin`,
// `absolutise`, `discover_links`, …) are ungated — they carry no ureq and their
// unit tests run in every build.
#[cfg(feature = "http-fetch")]
struct HtmlLinkDiscover;
#[cfg(feature = "http-fetch")]
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct HtmlLinkDiscoverConfig {
/// Index page to fetch (http/https).
url: String,
/// Regex tested (unanchored) against each raw `href` value; matches are kept.
pattern: String,
/// Destination URL-list path (newline-delimited), relative to the manifest dir.
out: String,
/// Extra request headers. A default `User-Agent` is set unless overridden here.
#[serde(default)]
headers: BTreeMap<String, String>,
}
#[cfg(feature = "http-fetch")]
impl HtmlLinkDiscoverConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("html_link_discover: invalid `with:` config: {}", e))
})
}
/// Compile `pattern`, mapping a bad regex to a load-time validation error.
fn compiled_pattern(&self) -> Result<regex::Regex> {
regex::Regex::new(&self.pattern).map_err(|e| {
Error::ManifestValidation(format!(
"html_link_discover: invalid `pattern` regex '{}': {}",
self.pattern, e
))
})
}
}
/// The origin (`scheme://authority`) of an absolute URL, or `None` if `url` isn't one.
fn url_origin(url: &str) -> Option<String> {
let (scheme, rest) = url.split_once("://")?;
if scheme.is_empty() {
return None;
}
let authority = rest.split(['/', '?', '#']).next().unwrap_or("");
if authority.is_empty() {
return None;
}
Some(format!("{}://{}", scheme, authority))
}
/// The RFC-3986 directory base of a page URL — the URL up to and including the last
/// `/` of its path (query/fragment stripped) — that a bare relative href resolves
/// against. Falls back to `origin/` when the path has no slash.
fn url_dir_base(page_url: &str) -> Option<String> {
let origin = url_origin(page_url)?;
let rest = &page_url[origin.len()..];
let path = rest.split(['?', '#']).next().unwrap_or("");
match path.rfind('/') {
Some(i) => Some(format!("{}{}", origin, &path[..=i])),
None => Some(format!("{}/", origin)),
}
}
/// Whether `s` begins with a URL scheme (`http:`, `mailto:`, …) per RFC-3986.
fn href_has_scheme(s: &str) -> bool {
match s.find(':') {
None | Some(0) => false,
Some(i) => {
let scheme = &s[..i];
scheme
.chars()
.next()
.map(|c| c.is_ascii_alphabetic())
.unwrap_or(false)
&& scheme
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
}
}
}
/// Absolutise an `href` against the page URL. Handles absolute URLs (kept verbatim),
/// protocol-relative (`//host/…`), root-relative (`/path` → page origin), and
/// path-relative (resolved against the page's directory base). Returns `None` for
/// empty or fragment-only hrefs.
fn absolutise(page_url: &str, href: &str) -> Option<String> {
let h = href.trim();
if h.is_empty() || h.starts_with('#') {
return None;
}
if href_has_scheme(h) {
return Some(h.to_string());
}
if let Some(rest) = h.strip_prefix("//") {
let scheme = page_url.split("://").next()?;
if scheme.is_empty() || scheme == page_url {
return None;
}
return Some(format!("{}://{}", scheme, rest));
}
if h.starts_with('/') {
return Some(format!("{}{}", url_origin(page_url)?, h));
}
Some(format!("{}{}", url_dir_base(page_url)?, h))
}
/// Extract every `href` in `html` whose value matches `pattern`, absolutise it
/// against `page_url`, and return the list de-duplicated in document order. `limit`
/// caps the number of distinct URLs returned (the `head -n N` of fetch_ncen.sh).
fn discover_links(
html: &str,
page_url: &str,
pattern: ®ex::Regex,
limit: Option<usize>,
) -> Vec<String> {
// Quoted href forms only (`href="…"` / `href='…'`) — unquoted attributes are
// rare in the gov index pages this targets.
let href_re = regex::Regex::new(r#"(?i)href\s*=\s*["']([^"']*)["']"#)
.expect("static href regex is valid");
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for cap in href_re.captures_iter(html) {
let href = &cap[1];
if !pattern.is_match(href) {
continue;
}
let Some(abs) = absolutise(page_url, href) else {
continue;
};
if seen.insert(abs.clone()) {
out.push(abs);
if let Some(l) = limit
&& out.len() >= l
{
break;
}
}
}
out
}
#[cfg(feature = "http-fetch")]
impl Operator for HtmlLinkDiscover {
fn name(&self) -> &'static str {
"html_link_discover"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = HtmlLinkDiscoverConfig::parse(with)?;
cfg.compiled_pattern()?; // fail fast on a bad regex at manifest load
// The network source is not a graph node; only the local URL list is.
let mut assets = OpAssets::default();
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
use crate::ingress_meta::DEFAULT_UA;
let cfg = HtmlLinkDiscoverConfig::parse(with)?;
let pattern = cfg.compiled_pattern()?;
let out = ctx.dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
// Default UA first, then Protocol overrides (a `User-Agent` key wins).
let mut req = ureq::get(&cfg.url).set("User-Agent", DEFAULT_UA);
for (k, v) in &cfg.headers {
req = req.set(k, v);
}
let resp = req
.call()
.map_err(|e| fetch_failed(format!("html_link_discover: GET {}: {}", cfg.url, e)))?;
let body = resp
.into_string()
.map_err(|e| fetch_failed(format!("html_link_discover: read {}: {}", cfg.url, e)))?;
let links = discover_links(&body, &cfg.url, &pattern, None);
let mut payload = String::new();
for l in &links {
payload.push_str(l);
payload.push('\n');
}
// Atomic write: sibling `.part`, then rename — a killed run never leaves a
// half-written list that looks complete.
let mut tmp_os = out.clone().into_os_string();
tmp_os.push(".part");
let tmp = PathBuf::from(tmp_os);
std::fs::write(&tmp, payload.as_bytes()).map_err(|e| {
fetch_failed(format!(
"html_link_discover: write {}: {}",
tmp.display(),
e
))
})?;
std::fs::rename(&tmp, &out).map_err(|e| {
fetch_failed(format!(
"html_link_discover: rename {}: {}",
out.display(),
e
))
})?;
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// archive_extract (ingress) — pull named / pattern-matched members out of a `.zip`
// into `dest` as first-class produced assets. The extraction half of the N-CEN
// pull: each quarter's `ncen_YYYYqN.zip` yields REGISTRANT.tsv and
// FUND_REPORTED_INFO.tsv (the CIK↔LEI / SERIES_ID↔LEI carriers).
//
// Three properties `command: unzip` doesn't give cleanly:
// • zip-slip guard — a member named `../…` or an absolute path is rejected, so a
// hostile archive can't write outside `dest`.
// • atomic per-file write — each member lands via a sibling `.part` + rename, so a
// killed run never leaves a half-written TSV that looks complete.
// • case-preserving on disk, case-folded in the graph — the file keeps its real
// name (`REGISTRANT.tsv`, referenced verbatim by downstream SQL) while the
// AssetGraph node name is lowercased, matching every other operator.
//
// zip dep: default-features OFF + `deflate-flate2` ONLY. The N-CEN zips are DEFLATE-
// compressed, and plain `deflate` is a zip base meta-feature with no inflate backend
// (→ runtime failure); `deflate-flate2` wires flate2 (already an arcform dep).
// Dropping defaults avoids pulling the bzip2/zstd/aes C backends.
// ─────────────────────────────────────────────────────────────────────────────
struct ArchiveExtract;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct ArchiveExtractConfig {
/// Source `.zip`, relative to the manifest dir (a `reads` asset).
archive: String,
/// Explicit member names to extract (exact match on the zip-internal name).
#[serde(default)]
members: Vec<String>,
/// Regex tested (unanchored) against each member name; matches are extracted.
#[serde(default)]
pattern: Option<String>,
/// Destination directory, relative to the manifest dir. Members are written under
/// it preserving their real-case relative path.
dest: String,
}
impl ArchiveExtractConfig {
fn parse(with: &Value) -> Result<Self> {
let cfg: Self = serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("archive_extract: invalid `with:` config: {}", e))
})?;
if cfg.members.is_empty() && cfg.pattern.is_none() {
return Err(Error::ManifestValidation(
"archive_extract: specify at least one of `members` or `pattern`".to_string(),
));
}
Ok(cfg)
}
/// Compile `pattern` (if any), mapping a bad regex to a load-time validation error.
fn compiled_pattern(&self) -> Result<Option<regex::Regex>> {
match &self.pattern {
None => Ok(None),
Some(p) => regex::Regex::new(p).map(Some).map_err(|e| {
Error::ManifestValidation(format!(
"archive_extract: invalid `pattern` regex '{}': {}",
p, e
))
}),
}
}
}
/// Whether a zip member is selected: an exact name in `members`, or a `pattern` hit.
fn member_selected(name: &str, members: &[String], pattern: Option<®ex::Regex>) -> bool {
members.iter().any(|m| m == name) || pattern.map(|re| re.is_match(name)).unwrap_or(false)
}
/// Resolve a zip member name to a SAFE relative path (case preserved). Returns `None`
/// for a zip-slip attempt — an absolute path, a Windows drive, or any `..` component —
/// so nothing can be written outside `dest`.
fn safe_relative(name: &str) -> Option<PathBuf> {
if name.starts_with('/') || name.starts_with('\\') {
return None;
}
let b = name.as_bytes();
if b.len() >= 2 && b[0].is_ascii_alphabetic() && b[1] == b':' {
return None; // Windows drive-absolute (C:\…)
}
let mut rel = PathBuf::new();
for comp in name.split(['/', '\\']) {
match comp {
"" | "." => continue,
".." => return None,
c => rel.push(c),
}
}
if rel.as_os_str().is_empty() {
return None;
}
Some(rel)
}
/// Extract the selected members of a zip `reader` into `dest`: case-preserving on
/// disk, zip-slip guarded, atomic per-file write. Returns the written paths. Factored
/// out of `run` so the extraction is unit-testable against an in-memory zip.
fn extract_members<R: std::io::Read + std::io::Seek>(
reader: R,
dest: &Path,
members: &[String],
pattern: Option<®ex::Regex>,
) -> Result<Vec<PathBuf>> {
let mut archive = zip::ZipArchive::new(reader)
.map_err(|e| fetch_failed(format!("archive_extract: open zip: {}", e)))?;
let mut written = Vec::new();
for i in 0..archive.len() {
let mut entry = archive
.by_index(i)
.map_err(|e| fetch_failed(format!("archive_extract: read entry {}: {}", i, e)))?;
if entry.is_dir() {
continue;
}
let name = entry.name().to_string();
if !member_selected(&name, members, pattern) {
continue;
}
let rel = safe_relative(&name).ok_or_else(|| {
fetch_failed(format!(
"archive_extract: unsafe member path '{}' (zip-slip rejected)",
name
))
})?;
let target = dest.join(&rel);
if let Some(parent) = target.parent() {
std::fs::create_dir_all(parent).map_err(|e| {
fetch_failed(format!(
"archive_extract: mkdir {}: {}",
parent.display(),
e
))
})?;
}
// Atomic per-file write: sibling `<target>.part`, then rename.
let mut tmp_os = target.clone().into_os_string();
tmp_os.push(".part");
let tmp = PathBuf::from(tmp_os);
let mut f = std::fs::File::create(&tmp).map_err(|e| {
fetch_failed(format!("archive_extract: create {}: {}", tmp.display(), e))
})?;
std::io::copy(&mut entry, &mut f)
.map_err(|e| fetch_failed(format!("archive_extract: extract '{}': {}", name, e)))?;
let _ = f.sync_all();
drop(f);
std::fs::rename(&tmp, &target).map_err(|e| {
fetch_failed(format!(
"archive_extract: rename {}: {}",
target.display(),
e
))
})?;
written.push(target);
}
Ok(written)
}
impl Operator for ArchiveExtract {
fn name(&self) -> &'static str {
"archive_extract"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = ArchiveExtractConfig::parse(with)?;
cfg.compiled_pattern()?; // fail fast on a bad regex at manifest load
// Returned case-preserved, exactly as `members` names them — this is the one
// place a zip member's real, possibly mixed case is known before extraction
// ever runs. `asset.rs` lowercases for graph identity and keeps this raw
// spelling alongside it for on-disk resolution (see its `declared_case`).
// Explicit `members` give per-file produced nodes; a pattern-only selection
// isn't known until the archive is opened, so `dest` stands in as the coarse
// produced node.
let mut assets = OpAssets::default();
assets.record_reads(cfg.archive.clone(), crate::asset_kind::AssetKind::File);
if cfg.members.is_empty() {
// Pattern-only selection isn't known until the archive is opened, so
// `dest` stands in as the coarse produced node — and it really is a
// directory: extraction writes however many files the pattern matches
// into it, not one file under that name.
assets.record_produces(cfg.dest.clone(), crate::asset_kind::AssetKind::Directory);
} else {
// Explicit `members` give per-file produced nodes, case-preserved
// exactly as `members` names them.
let base = cfg.dest.trim_end_matches('/');
for m in &cfg.members {
assets.record_produces(
format!("{}/{}", base, m),
crate::asset_kind::AssetKind::File,
);
}
}
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = ArchiveExtractConfig::parse(with)?;
let pattern = cfg.compiled_pattern()?;
let archive_path = ctx.dir.join(&cfg.archive);
let dest = ctx.dir.join(&cfg.dest);
std::fs::create_dir_all(&dest).map_err(|e| {
fetch_failed(format!("archive_extract: mkdir {}: {}", dest.display(), e))
})?;
let file = std::fs::File::open(&archive_path).map_err(|e| {
fetch_failed(format!(
"archive_extract: open {}: {}",
archive_path.display(),
e
))
})?;
extract_members(file, &dest, &cfg.members, pattern.as_ref())?;
Ok(StepOutput {
stderr: String::new(),
stdout: None,
report: None,
})
}
}
// ─────────────────────────────────────────────────────────────────────────────
// splink_resolve (transform) — the probabilistic EDGAR↔GLEIF name match, promoted
// from the opaque `command: uv run --script …/resolve.py …` step to a first-class,
// versioned operator. `op: splink_resolve@1` addresses the EXACT frozen script bytes
// embedded below (SEED=42 Fellegi-Sunter model; see resolve.py) — the reproducibility
// contract the "call the script by relative path" step lacked. Its `reads`/`produces`
// put the resolve *in* the asset graph, so a stale input propagates instead of the
// graph-island silent-skip the path-based step risked.
//
// Wraps the arcform-EMBEDDED resolve.py via the uv-run substrate (materialize_frozen_
// script → uv_run_args → run_process), NOT a Rust reimplementation of Splink — Python
// at the edges, same posture as datapackage_describe. Runs in OutputMode::Inherit so
// resolve.py's `[coverage]` / `[hero tickers]` tables stream live and the step timeout
// (the manifest's `timeout_sec`) is enforced by wait_with_timeout.
//
// SCALE — this is the ~4 h full-corpus job (GLEIF ≈ 3.2M rows). The manifest step MUST
// pin `retry.max_attempts: 1`: a non-zero exit or timeout is retryable in the substrate,
// so without a cap the engine would re-attempt a 4 h job up to `defaults.retry`. (A
// spawn failure — bad `uv`/binary — is already NON-retryable, so a typo never burns
// attempts; but a mid-run failure is the case the manifest cap guards.) `--sample` is a
// smoke-test cap on GLEIF rows and is OMITTED for a published run (see the arg builder).
// ─────────────────────────────────────────────────────────────────────────────
/// The resolve script, pinned into the binary. `@1` == these exact bytes.
const RESOLVE_PY: &str = include_str!("../operators/splink_resolve/resolve.py");
/// The script cache directory `materialize_frozen_script` builds for this operator.
/// Test-only, for the same reason [`UMAP_PROJECT_CACHE_DIR`] is: production never
/// spells the path, and a test traps it to drive the cache's two failure paths.
#[cfg(test)]
const SPLINK_RESOLVE_CACHE_DIR: &str = "arcform-op-splink_resolve-1.0.0";
struct SplinkResolve;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct SplinkResolveConfig {
/// EDGAR / SEC-entity Parquet (cik, ticker, company_name) — the crosswalk's left
/// side. A `reads` asset, resolved against ctx.dir.
edgar: String,
/// GLEIF golden-copy Parquet (lei, legal_name, jurisdiction, country, reg_status) —
/// the right side. A `reads` asset, resolved against ctx.dir.
gleif: String,
/// Resolved-crosswalk Parquet to write (the `produces` asset). Resolved against ctx.dir.
out: String,
/// Optional GLEIF-row cap for a fast smoke test (reservoir sample, SEED=42). OMITTED
/// from the argv unless `Some(n)` with `n > 0`; a published run leaves it unset (or 0)
/// and resolves the full corpus.
#[serde(default)]
sample: Option<u64>,
}
impl SplinkResolveConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("splink_resolve: invalid `with:` config: {}", e))
})
}
}
/// Build resolve.py's argv tail (everything after the script path). `--sample` is emitted
/// ONLY for a positive cap: a published run passes `sample: None` (or `0`) and the full
/// GLEIF corpus is resolved. Factored out so the omit-on-publish rule is unit-testable
/// without spawning `uv`.
fn splink_resolve_args(edgar: &str, gleif: &str, out: &str, sample: Option<u64>) -> Vec<String> {
let mut a = vec![
"--edgar".to_string(),
edgar.to_string(),
"--gleif".to_string(),
gleif.to_string(),
"--out".to_string(),
out.to_string(),
];
if let Some(n) = sample
&& n > 0
{
a.push("--sample".to_string());
a.push(n.to_string());
}
a
}
impl Operator for SplinkResolve {
fn name(&self) -> &'static str {
"splink_resolve"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = SplinkResolveConfig::parse(with)?;
let mut assets = OpAssets::default();
assets.record_reads(cfg.edgar.clone(), crate::asset_kind::AssetKind::File);
assets.record_reads(cfg.gleif.clone(), crate::asset_kind::AssetKind::File);
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = SplinkResolveConfig::parse(with)?;
let out = ctx.dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
let args = splink_resolve_invocation(&cfg, ctx.dir)?;
// Inherit: stream resolve.py's coverage/hero-ticker tables live and honour the
// step timeout (a 4 h job that overruns is killed, not left hanging).
run_process("uv", &args, ctx, OutputMode::Inherit, "splink_resolve")
}
}
/// The full argv `run` hands `uv`: materialise the frozen script and build its
/// arguments. Split out of [`Operator::run`], as [`umap_project_invocation`] is, so a
/// test reads the argv the script is really given without spawning `uv`.
fn splink_resolve_invocation(cfg: &SplinkResolveConfig, dir: &Path) -> Result<Vec<String>> {
let script = materialize_frozen_script("splink_resolve", "1.0.0", RESOLVE_PY)?;
let extra = splink_resolve_args(
&dir.join(&cfg.edgar).display().to_string(),
&dir.join(&cfg.gleif).display().to_string(),
&dir.join(&cfg.out).display().to_string(),
cfg.sample,
);
Ok(uv_run_args(&script.to_string_lossy(), &extra))
}
// ─────────────────────────────────────────────────────────────────────────────
// gleif_ra_fetch (ingress) — page the public GLEIF v1 API for every entity
// registered under one registration authority (RA000665 = the SEC) and emit the
// `lei,registered_as,category` CSV the `load` step consumes.
//
// Retires `edgar_gleif/scripts/fetch_gleif_ra.sh`. That script hand-rolled cursor
// paging in bash (`page[cursor]=*`, follow `.links.next`, terminate on null/empty,
// with a loop guard) because GLEIF hard-caps OFFSET paging at 10,000 records and
// RA000665 has ~27k. The embedded `fetch_gleif_ra.py` does the same with dlt's
// `RESTClient` + `JSONLinkPaginator(next_url_path="links.next")` — STATELESS
// full-refresh (no incremental cursor persisted) — then writes a DETERMINISTIC
// SORTED CSV so the row-set parity gate against the retired script is a clean
// `EXCEPT`. Same uv-run substrate + frozen-script contract as datapackage_describe
// and splink_resolve: `op@1` addresses these exact script bytes.
//
// Ingress, so `reads: []` (the network source is not a graph node) and
// `produces: [out]` — the CSV sits in the AssetGraph downstream of nothing and
// upstream of `load`, exactly where the old `command:` step sat but now typed.
// ─────────────────────────────────────────────────────────────────────────────
/// The GLEIF-fetch script, pinned into the binary. `@1` == these exact bytes.
const GLEIF_RA_FETCH_PY: &str = include_str!("../operators/gleif_ra_fetch/fetch_gleif_ra.py");
/// The script cache directory `materialize_frozen_script` builds for this operator.
/// Test-only, for the same reason [`UMAP_PROJECT_CACHE_DIR`] is: production never
/// spells the path, and a test traps it to drive the cache's two failure paths.
#[cfg(test)]
const GLEIF_RA_FETCH_CACHE_DIR: &str = "arcform-op-gleif_ra_fetch-1.0.0";
struct GleifRaFetch;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct GleifRaFetchConfig {
/// GLEIF registration-authority id to page (e.g. `RA000665`, the SEC).
ra: String,
/// Destination CSV path, relative to the manifest directory (the `produces`
/// asset). The script writes it atomically (`.part` + rename).
out: String,
/// GLEIF `page[size]`. Defaults to the script's 200 when omitted.
#[serde(default)]
page_size: Option<u64>,
/// User-Agent request header. Defaults to the script's Meridian UA when omitted.
#[serde(default)]
user_agent: Option<String>,
}
impl GleifRaFetchConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("gleif_ra_fetch: invalid `with:` config: {}", e))
})
}
}
impl Operator for GleifRaFetch {
fn name(&self) -> &'static str {
"gleif_ra_fetch"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = GleifRaFetchConfig::parse(with)?;
// Ingress: the network source is not a graph node; only the local CSV is.
let mut assets = OpAssets::default();
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = GleifRaFetchConfig::parse(with)?;
let out = ctx.dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
let args = gleif_ra_fetch_invocation(&cfg, ctx.dir)?;
run_process("uv", &args, ctx, OutputMode::Capture, "gleif_ra_fetch")
}
}
/// The full argv `run` hands `uv`: materialise the frozen script and build its
/// arguments. Split out of [`Operator::run`], as [`umap_project_invocation`] is, so a
/// test reads the argv the script is really given without spawning `uv`.
fn gleif_ra_fetch_invocation(cfg: &GleifRaFetchConfig, dir: &Path) -> Result<Vec<String>> {
let script = materialize_frozen_script("gleif_ra_fetch", "1.0.0", GLEIF_RA_FETCH_PY)?;
let mut extra = vec![
"--ra".to_string(),
cfg.ra.clone(),
"--out".to_string(),
dir.join(&cfg.out).display().to_string(),
];
// `--max-pages` is intentionally NOT surfaced in config — it is a smoke-test
// lever only. Production always does a full, guarded, unbounded pull.
if let Some(ps) = cfg.page_size {
extra.push("--page-size".to_string());
extra.push(ps.to_string());
}
if let Some(ref ua) = cfg.user_agent {
extra.push("--user-agent".to_string());
extra.push(ua.clone());
}
Ok(uv_run_args(&script.to_string_lossy(), &extra))
}
// ─────────────────────────────────────────────────────────────────────────────
// umap_project (transform) — reduce numeric columns that ALREADY EXIST to the two
// coordinates a map is drawn from. Reads one Parquet, builds a feature matrix from
// the named columns, reduces it to two dimensions with UMAP, and writes a Parquet
// carrying every input column plus `projection_x`, `projection_y` and
// `projection_fit_id` — the last a fingerprint of the exact numbers and knobs one fit
// consumed, broadcast to every row, so two files can be compared for "same fit" before
// their positions are compared row for row. See operators/umap_project/README.md,
// "Telling a refit from an append."
//
// `fit:` IS WHAT MAKES THE MAP HOLD STILL, and it is the one field here that names an
// asset on BOTH sides of the step. Omit it and every run refits the whole input, so
// appending one row moves every point and `projection_fit_id` changes to say so. Set
// it and the first run writes the fitted projection to that path while every later
// run reads it back: rows the fit already holds keep their exact coordinates, and
// appended rows are placed into that layout with `UMAP.transform` instead of starting
// a new one.
//
// SO THE STEP BOTH `produces` THE FIT AND `reads` IT, and that declaration is what
// earns the caching rather than decorating it. The asset graph carries the shape
// already: `AssetGraph::validate_order` treats a name a step reads and produces as a
// self-contained operation rather than an ordering violation, `downstream_steps` will
// not drag a step downstream of itself, and `produced_artifact_hash` sorts and dedups
// the names so the fit is hashed once. What the declaration buys is that the fit's
// BYTES are part of what the step is answerable for: delete or corrupt it and the
// artifact hash moves, which marks the step stale and refits; leave it alone with the
// input unchanged and the step is hash-clean and skips without spawning `uv` at all.
// Undeclared, the fit would be a file arc could not see, and a deleted one would look
// exactly like a fresh run.
//
// A FIT THAT NO LONGER MATCHES THE INPUT IS REFUSED, naming the mismatch, rather than
// placing rows into a layout built for different columns. That covers a manifest edit
// too: `neighbors:`, `min_dist:` and `metric:` are part of what a fit IS, so moving
// one marks the step stale, the run finds a fit built under the old value, and it
// stops. Refitting under new knobs means deleting the fit — an analyst's decision,
// and not one a re-run gets to make on their behalf by moving every point.
//
// NO TEXT COLUMN AND NO MODEL, and that is the whole shape of it. `columns:` names
// columns that are already numbers — a numeric scalar contributes one feature, a
// list/array of numerics contributes one per element — so `[longitude, latitude]`
// maps a table of places and `[embedding]` maps whatever wrote a vector column,
// without this operator knowing which produced it. A step that insisted on embedding
// text first could serve neither case alone.
//
// THE TIER, stated here because reaching past one requires naming what above cannot
// do the job. arcform builds a capability as a DuckDB extension first, then in
// Rust/C, and reaches a managed environment (`uv`) only when the tiers above cannot
// carry the work. This is the third tier and the reason is UMAP: `umap-learn` is the
// reference implementation, there is no DuckDB extension for it and no Rust one in
// this stack, and a reimplementation would emit different coordinates from every
// published UMAP map anyone might compare this one against. The capability is absent
// above, not merely inconvenient.
//
// Same uv-run substrate + frozen-script contract as the other Python operators:
// `op@1` addresses these exact script bytes, so the seed and the projection
// parameters cannot drift under a manifest.
//
// OutputMode::Capture, not Inherit: the script reports a fixable condition (no such
// column, a column that is not a number, a column-name collision) as one line on
// stderr, and Capture is what carries that line into the run's error. The cost is
// that a long projection prints nothing until it finishes.
// ─────────────────────────────────────────────────────────────────────────────
/// The projection script, pinned into the binary. `@1` == these exact bytes.
const UMAP_PROJECT_PY: &str = include_str!("../operators/umap_project/umap_project.py");
/// The script cache directory `materialize_frozen_script` builds for this operator,
/// under the system temp directory. Test-only, because production never spells the
/// path — it passes the name and version and lets the helper build it. A test traps
/// that path to drive the cache's two failure paths, and if the helper ever spells it
/// differently the trap stops biting, materialising succeeds, and the test goes RED on
/// its own `expect_err`. It cannot drift quietly.
#[cfg(test)]
const UMAP_PROJECT_CACHE_DIR: &str = "arcform-op-umap_project-1.2.0";
/// The metrics a manifest may ask for, and the single place they are written down:
/// [`with_schema`] emits this array as the field's `enum`, so an authoring form and
/// the load-time refusal below cannot disagree about what is accepted.
///
/// umap-learn accepts many more. These two are the ones the suite drives, and they
/// are the two readings that matter: `euclidean` for an arbitrary feature matrix
/// (umap-learn's own default), `cosine` for L2-normalised vectors, which is what an
/// embedding is. A name outside the set is refused at manifest load rather than by an
/// exception from inside UMAP an hour into a run.
const UMAP_METRICS: [&str; 2] = ["euclidean", "cosine"];
struct UmapProject;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct UmapProjectConfig {
/// Parquet holding the columns to project (a `reads` asset). Resolved against
/// ctx.dir.
input: String,
/// The numeric columns to reduce. Whether a named column exists, and whether its
/// type is one that can be placed on a map, is the script's refusal rather than a
/// load-time one — the manifest cannot know the schema.
columns: Vec<String>,
/// Parquet to write: every input column, plus `projection_x` and `projection_y`
/// as DOUBLE and `projection_fit_id` as VARCHAR (the `produces` asset). Resolved
/// against ctx.dir.
out: String,
/// Where the fitted projection is kept between runs — declared as an asset this
/// step BOTH reads and produces, which is what lets a second run find it instead
/// of refitting. Written by the first run, read by every later one. OMITTED when
/// unset, in which case the operator persists nothing and every run refits the
/// whole input, which is the behaviour every Protocol had before this field
/// existed. Resolved against ctx.dir.
#[serde(default)]
fit: Option<String>,
/// UMAP's `n_neighbors` — how much of the input each point is placed against.
/// Low reads local structure, high reads global. OMITTED from the argv when
/// unset, so the script's own default stands.
#[serde(default)]
neighbors: Option<u64>,
/// UMAP's `min_dist` — how tightly points may pack. Omitted from the argv when
/// unset. Must be in `[0, 1)`: at or above the default spread of 1.0 the layout
/// has no room to separate anything.
#[serde(default)]
min_dist: Option<f64>,
/// How distance between two rows is measured. Omitted from the argv when unset.
/// One of [`UMAP_METRICS`].
#[serde(default)]
metric: Option<String>,
}
impl UmapProjectConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("umap_project: invalid `with:` config: {}", e))
})
}
/// Refuse, at manifest load rather than an hour into a run, everything about this
/// config that is decidable without reading the input's schema. Both UMAP bounds
/// are UMAP's, not this operator's invention: a neighbourhood of fewer than two
/// points has no neighbourhood, and `min_dist` is a fraction of the layout's
/// spread.
fn validate(&self) -> Result<()> {
if self.columns.is_empty() {
return Err(Error::ManifestValidation(
"umap_project: `columns:` is empty — there is nothing to project. Name \
the numeric columns the map is drawn from."
.to_string(),
));
}
for (at, column) in self.columns.iter().enumerate() {
if self.columns[..at].contains(column) {
return Err(Error::ManifestValidation(format!(
"umap_project: `columns:` names {:?} more than once — each column \
contributes its features once",
column
)));
}
}
if let Some(n) = self.neighbors
&& n < 2
{
return Err(Error::ManifestValidation(format!(
"umap_project: `neighbors: {}` is below 2 — a point placed against \
fewer than two neighbours has no neighbourhood to be placed in",
n
)));
}
if let Some(d) = self.min_dist
&& !(0.0..1.0).contains(&d)
{
return Err(Error::ManifestValidation(format!(
"umap_project: `min_dist: {}` is outside [0, 1) — it is a fraction of \
the layout's spread, so at 1.0 and above nothing can separate",
d
)));
}
if let Some(m) = &self.metric
&& !UMAP_METRICS.contains(&m.as_str())
{
return Err(Error::ManifestValidation(format!(
"umap_project: `metric: {}` is not one this operator will pass to UMAP \
— it accepts {}",
m,
UMAP_METRICS.join(" or ")
)));
}
Ok(())
}
/// Refuse a `neighbors:` above one below the input's row count, when the step
/// runs and before anything is written.
///
/// Not part of [`Self::validate`], because the manifest cannot know the row count:
/// an earlier step may write the input. So the input is counted here, after every
/// step before this one has run, and the refusal comes ahead of
/// [`umap_project_invocation`], which creates the output's directory and the
/// fit's. Above the bound the script would clamp the value and draw the same map
/// as the bound itself, exiting 0; refusing is what makes the value mean what it
/// says. The script keeps its clamp as a second guard, for an unset value on a
/// small table and for a run of the script outside arc.
///
/// Only a value the manifest SET is counted against: an unset one takes the
/// script's 15 or the bound, whichever is smaller, which is the rule the schema
/// states. And an input that cannot be read is not refused here — the script
/// reads it next and fails naming it.
///
/// `ManifestValidation`, so the runner halts on it without a retry and `arc run`
/// exits 1, as it does for a `neighbors:` below 2.
fn refuse_neighbors_past_the_input(&self, dir: &Path) -> Result<()> {
let Some(asked) = self.neighbors else {
return Ok(());
};
let Some(rows) = parquet_row_count(&dir.join(&self.input)) else {
return Ok(());
};
let bound = rows.saturating_sub(1);
if asked > bound {
return Err(Error::ManifestValidation(format!(
"umap_project: `neighbors: {}` is above {}, one below the {} rows of {} \
— UMAP cannot place a point against more rows than there are besides \
it. Set `neighbors:` to {} or less, or leave it unset to take 15 or {}, \
whichever is smaller",
asked, bound, rows, self.input, bound, bound
)));
}
Ok(())
}
}
/// The number of rows in the Parquet at `path`, read in process, or `None` when it
/// cannot be read. It counts what `umap_project.py` counts: every row `read_parquet`
/// returns for the same path.
fn parquet_row_count(path: &Path) -> Option<u64> {
let conn = duckdb::Connection::open_in_memory().ok()?;
let rows: i64 = conn
.query_row(
&format!(
"SELECT count(*) FROM read_parquet('{}')",
sql_lit(&path.display().to_string())
),
[],
|row| row.get(0),
)
.ok()?;
u64::try_from(rows).ok()
}
/// Build umap_project.py's argv tail (everything after the script path). One
/// `--column` per projected column, in the order the manifest listed them, then the
/// three optional knobs — emitted ONLY when the manifest set them, so the frozen
/// script's own defaults stay the single place they are written down. Factored out so
/// that rule — and the arg order — is unit-testable without spawning `uv`.
fn umap_project_args(
input: &str,
columns: &[String],
out: &str,
fit: Option<&str>,
neighbors: Option<u64>,
min_dist: Option<f64>,
metric: Option<&str>,
) -> Vec<String> {
let mut a = vec!["--input".to_string(), input.to_string()];
for column in columns {
a.push("--column".to_string());
a.push(column.clone());
}
a.push("--out".to_string());
a.push(out.to_string());
if let Some(f) = fit {
a.push("--fit".to_string());
a.push(f.to_string());
}
if let Some(n) = neighbors {
a.push("--neighbors".to_string());
a.push(n.to_string());
}
if let Some(d) = min_dist {
a.push("--min-dist".to_string());
a.push(d.to_string());
}
if let Some(m) = metric {
a.push("--metric".to_string());
a.push(m.to_string());
}
a
}
impl Operator for UmapProject {
fn name(&self) -> &'static str {
"umap_project"
}
// 1.2.0, not 1.1.0: the operator gained `fit:`, which is a new asset edge and a
// new flag on the frozen script, and `op@<version>` addressing exact bytes means a
// behaviour change is a version bump and a rebuild, never a silent edit (see
// `materialize_frozen_script`). Additive and backward compatible — a manifest
// that sets no `fit:` gets exactly the argv, the assets and the map it got at
// 1.1.0 — so a minor bump; `@1` in a manifest still resolves it. 1.1.0 was the
// bump for `projection_fit_id`, the output column that says WHICH fit a file was
// drawn from; this one is the fit itself.
fn version(&self) -> semver::Version {
semver::Version::new(1, 2, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = UmapProjectConfig::parse(with)?;
cfg.validate()?;
let mut assets = OpAssets::default();
assets.record_reads(cfg.input.clone(), crate::asset_kind::AssetKind::File);
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
// The fit is declared on BOTH sides, and the two sides do NOT split the work
// evenly — this comment used to say each carried half of it, and deleting one
// line at a time is how that was found to be wrong. EITHER side alone puts the
// fit's bytes into the step's artifact hash, because `produced_artifact_hash`
// folds in what a step produces AND the reads that nothing in the manifest
// produces; drop `record_produces` here and the staleness behaviour is
// unchanged. What each side actually buys:
//
// `reads` is the lineage — this is a file the step CONSUMES, and it is the
// side that would order this step after another if a Protocol ever had one
// step write a fit that a second places rows into.
//
// `produces` is answerability. It is what `missing_declared_produces` reads,
// so a run that "succeeds" without leaving a fit on disk says so by name
// instead of exiting 0 in silence — the shape of an operator that took the
// field and ignored it. Read-only, that run is indistinguishable from one
// whose fit was simply an input it did not have to make.
//
// See this module's `fit:` paragraph for why the graph carries one name on
// both sides without contradiction.
if let Some(fit) = &cfg.fit {
assets.record_reads(fit.clone(), crate::asset_kind::AssetKind::File);
assets.record_produces(fit.clone(), crate::asset_kind::AssetKind::File);
}
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = UmapProjectConfig::parse(with)?;
cfg.validate()?;
cfg.refuse_neighbors_past_the_input(ctx.dir)?;
let args = umap_project_invocation(&cfg, ctx.dir)?;
run_process("uv", &args, ctx, OutputMode::Capture, "umap_project")
}
}
/// Everything that has to be decided BEFORE `uv` is spawned: materialise the frozen
/// script and build the full argv.
///
/// Split out of [`Operator::run`] rather than left inline, and the reason is that the
/// projection needs a `uv` on PATH while the decision to attempt it does not. Inline,
/// every one of these lines was reachable only through a test that runs Python — so on
/// a machine without `uv`, which is every CI runner here, nothing checked that the
/// script materialised at all, that `op@1` addressed the embedded bytes, or that an
/// out-of-range knob stopped the step. As a function it is driven directly, argv and
/// all, by a test that spawns nothing.
fn umap_project_invocation(cfg: &UmapProjectConfig, dir: &Path) -> Result<Vec<String>> {
let input = dir.join(&cfg.input);
let out = dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
// The fit gets the same treatment as the output, and for the same reason: the
// script is asked to WRITE this path on a first run, so a `fit:` under a `build/`
// that no step has created yet must not fail on the directory rather than on
// anything about the projection. `None` when the manifest set no `fit:`, which is
// the whole of what "this operator persists nothing" now means.
let fit = cfg.fit.as_ref().map(|f| dir.join(f));
if let Some(parent) = fit.as_ref().and_then(|f| f.parent()) {
let _ = std::fs::create_dir_all(parent);
}
let script = materialize_frozen_script("umap_project", "1.2.0", UMAP_PROJECT_PY)?;
let extra = umap_project_args(
&input.display().to_string(),
&cfg.columns,
&out.display().to_string(),
fit.as_ref().map(|f| f.display().to_string()).as_deref(),
cfg.neighbors,
cfg.min_dist,
cfg.metric.as_deref(),
);
Ok(uv_run_args(&script.to_string_lossy(), &extra))
}
// ─────────────────────────────────────────────────────────────────────────────
// text_embed (transform) — PROVISIONAL. ON ITS WAY OUT. DO NOT HARDEN IT.
//
// Embeds a text column with the DuckDB embedding extension and writes a Parquet
// carrying every input column plus a vector column of FLOAT. It writes vectors and
// nothing else, which is the point of it being its own step: an analyst who wants
// vectors for similarity, clustering, deduplication or classifier features stops
// here, and one who wants a map feeds the vector column to `umap_project`, which
// neither knows nor cares that an embedding produced it.
//
// IT NO LONGER COMPUTES AN EMBEDDING, and that is the change worth knowing about.
// The script used to tokenise, index a `[vocab, dim]` matrix, mean-pool and
// L2-normalise in Python, which made one capability exist twice in two languages.
// The two disagreed — over the corpus in `tests/text_embed_parity.rs`, against the
// same weights, the vectors agreed to float32 summation order for ordinary text and
// departed for text carrying tokens the vocabulary does not have and for text past
// the truncation boundary, and the Python side was the one that departed from the
// reference implementation both claimed to follow. No magnitude is quoted: the Python
// path is deleted, so nothing regenerates a difference against it and no test reddens
// when a figure written here rots. The computation is deleted rather than corrected,
// so a SQL `subtoken_embed()` call and a Protocol run agree by construction instead of by
// tolerance.
//
// WHERE THIS IS GOING, and why nothing here should grow. What is left is a
// Parquet-in/Parquet-out wrapper around one SQL statement, which is a step a Protocol
// can write for itself. It survives only because the extension is a file a Protocol
// has to carry rather than something installable; when that changes, this operator is
// DELETED rather than maintained. A knob proposed here belongs on the extension.
//
// THE EXTENSION IS A DECLARED READ, NOT A DOWNLOAD, and that is the whole reason this
// belongs in an asset-centric engine rather than beside one. An artifact the operator
// fetched invisibly would be a graph edge that does not appear in the graph: the
// Protocol would depend on bytes it never names, a re-run could silently embed
// against a different build, and the run's cost would include a transfer nobody
// declared. Declared as a read, it is hashed for staleness like any other input —
// swap the extension and every downstream step re-runs — and `run` refuses before it
// spawns `uv` when it is not on disk, naming the file that is missing.
//
// THE MODEL IS CHECKED, NOT LOADED. The weights live inside the extension binary, so
// there is no directory to read them from and `model:` is optional. When it is set it
// declares WHICH model the Protocol believes it is embedding with, and the script
// refuses unless the extension's own content address over its bundled assets is
// reproduced by the declared directory. That address covers three files, so `run`
// refuses before `uv` when the directory carries fewer.
// ─────────────────────────────────────────────────────────────────────────────
/// The embedding script, pinned into the binary. `@1` == these exact bytes.
const TEXT_EMBED_PY: &str = include_str!("../operators/text_embed/text_embed.py");
/// The script cache directory `materialize_frozen_script` builds for this operator.
/// Test-only, for the same reason [`UMAP_PROJECT_CACHE_DIR`] is: production never
/// spells the path, and a test traps it to drive the cache's two failure paths.
#[cfg(test)]
const TEXT_EMBED_CACHE_DIR: &str = "arcform-op-text_embed-1.0.0";
/// The three files a published `model2vec` / `potion` release ships, and the three
/// the extension's model address is computed over. Checked here, before `uv` is
/// spawned, so a Protocol pointed at a directory that cannot reproduce that address
/// fails in milliseconds naming the missing file instead of after a dependency
/// resolve. TWO of these are not enough: a directory holding only the weights and the
/// tokenizer produces a different digest, which is indistinguishable from the wrong
/// model — so the missing file is named rather than the check being weakened.
const MODEL_PARTS: [&str; 3] = ["tokenizer.json", "model.safetensors", "config.json"];
struct TextEmbed;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct TextEmbedConfig {
/// Parquet holding the corpus (a `reads` asset). Resolved against ctx.dir.
input: String,
/// The column to embed. A column that is not in the Parquet is the script's
/// refusal, not a load-time one — the manifest cannot know the schema.
text_column: String,
/// The loadable embedding-extension artifact (a `reads` asset). Resolved against
/// ctx.dir. The Protocol is what puts it there; this operator never downloads it,
/// and never installs one from a registry.
extension: String,
/// Parquet to write: every input column, plus the vector column (the `produces`
/// asset). Resolved against ctx.dir.
out: String,
/// The column the vector is written into. OMITTED from the argv when unset, so
/// the script's own default stands. Name another to embed a second column of the
/// same corpus in a later step without colliding with this one's output.
#[serde(default)]
vector_column: Option<String>,
/// A model directory to CHECK the extension against (a `reads` asset, Directory
/// kind). Optional, and never read for weights — the weights are inside the
/// extension. Set with `model_release`, or not at all.
#[serde(default)]
model: Option<String>,
/// `<model-id>@<revision>` naming the release `model` was taken from. The
/// extension hashes both into its content address, so the address cannot be
/// reproduced from the directory alone and this is not a label.
#[serde(default)]
model_release: Option<String>,
}
impl TextEmbedConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone()).map_err(|e| {
Error::ManifestValidation(format!("text_embed: invalid `with:` config: {}", e))
})
}
fn validate(&self) -> Result<()> {
if let Some(name) = &self.vector_column
&& name.trim().is_empty()
{
return Err(Error::ManifestValidation(
"text_embed: `vector_column:` is blank — it names the column the \
vectors are written into, so it has to be a name."
.to_string(),
));
}
if self.model.is_some() != self.model_release.is_some() {
return Err(Error::ManifestValidation(
"text_embed: `model:` and `model_release:` are checked against each \
other and against the extension, so neither means anything alone. \
Set both, or neither."
.to_string(),
));
}
Ok(())
}
}
/// Build text_embed.py's argv tail (everything after the script path). The optional
/// vector-column name is emitted ONLY when the manifest set it, so the frozen
/// script's own default stays the single place it is written down. Factored out so
/// that rule — and the arg order — is unit-testable without spawning `uv`.
fn text_embed_args(
input: &str,
text_column: &str,
extension: &str,
out: &str,
vector_column: Option<&str>,
model: Option<(&str, &str)>,
) -> Vec<String> {
let mut a = vec![
"--input".to_string(),
input.to_string(),
"--text-column".to_string(),
text_column.to_string(),
"--extension".to_string(),
extension.to_string(),
"--out".to_string(),
out.to_string(),
];
if let Some(name) = vector_column {
a.push("--vector-column".to_string());
a.push(name.to_string());
}
if let Some((dir, release)) = model {
a.push("--model".to_string());
a.push(dir.to_string());
a.push("--model-release".to_string());
a.push(release.to_string());
}
a
}
/// What is missing from the declared model asset, or `None` when it is all there.
///
/// Answers with the path that is absent rather than a bare bool, because that path is
/// the whole message: the fix for "the model is not here" is to run the step that
/// puts it here, and the reader needs to know which file the operator looked for.
/// A `model` that is not a directory is left to the script — the directory layout is
/// this operator's contract, and a caller pointing at something else deserves the
/// script's reading of it rather than a guess made here.
fn missing_model_part(model: &Path) -> Option<PathBuf> {
if !model.exists() {
return Some(model.to_path_buf());
}
if model.is_dir() {
return MODEL_PARTS
.iter()
.map(|part| model.join(part))
.find(|part| !part.is_file());
}
None
}
impl Operator for TextEmbed {
fn name(&self) -> &'static str {
"text_embed"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = TextEmbedConfig::parse(with)?;
cfg.validate()?;
let mut assets = OpAssets::default();
assets.record_reads(cfg.input.clone(), crate::asset_kind::AssetKind::File);
// The extension is an input like any other, and it is the one that decides
// what a vector IS: swap the artifact and every vector downstream of it is
// from a different model, so it is hashed for staleness like the corpus.
assets.record_reads(cfg.extension.clone(), crate::asset_kind::AssetKind::File);
// The model directory, when declared, is read only to be hashed against what
// the extension reports. Directory kind, so editing a file in it re-runs the
// step rather than leaving a check that passed against different bytes.
if let Some(model) = &cfg.model {
assets.record_reads(model.clone(), crate::asset_kind::AssetKind::Directory);
}
assets.record_produces(cfg.out.clone(), crate::asset_kind::AssetKind::File);
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = TextEmbedConfig::parse(with)?;
cfg.validate()?;
let args = text_embed_invocation(&cfg, ctx.dir)?;
run_process("uv", &args, ctx, OutputMode::Capture, "text_embed")
}
}
/// Everything that has to be decided BEFORE `uv` is spawned: refuse a model that is
/// not on disk, materialise the frozen script, and build the full argv. Split out for
/// the same reason [`umap_project_invocation`] is — every CI runner here is a machine
/// without `uv`, and this is the half of `run` such a machine can still drive.
fn text_embed_invocation(cfg: &TextEmbedConfig, dir: &Path) -> Result<Vec<String>> {
let input = dir.join(&cfg.input);
let extension = dir.join(&cfg.extension);
let out = dir.join(&cfg.out);
if let Some(parent) = out.parent() {
let _ = std::fs::create_dir_all(parent);
}
// Before `uv`, and before anything that could be mistaken for a fetch or an
// install.
//
// NON-retryable (`StepExecution`, the missing-binary class) rather than
// `StepFailed`. An artifact that is not on disk will not be on disk on the second
// attempt either, so a Protocol carrying `defaults.retry` would otherwise pay its
// backoff three times over to be told the same thing.
if !extension.is_file() {
return Err(Error::StepExecution {
step: "text_embed".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!(
"the extension asset {} is not on disk — {} is missing. The \
embedding extension is a declared input, so a step in this \
Protocol has to put it there; this operator does not download \
one and does not install one from a registry.",
cfg.extension,
extension.display()
),
),
});
}
// The model directory, when declared, has to be able to reproduce the extension's
// content address, and it cannot do that with a file missing. Refused here so the
// reader is told WHICH file rather than being told the address did not match.
let model = cfg.model.as_ref().map(|m| dir.join(m));
if let (Some(name), Some(path)) = (&cfg.model, &model)
&& let Some(part) = missing_model_part(path)
{
return Err(Error::StepExecution {
step: "text_embed".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::NotFound,
format!(
"the declared model asset {} cannot be checked against the \
extension — {} is missing. The extension's model address covers \
{}, so a directory carrying fewer than all three cannot \
reproduce it. This operator does not download one.",
name,
part.display(),
MODEL_PARTS.join(", ")
),
),
});
}
let script = materialize_frozen_script("text_embed", "1.0.0", TEXT_EMBED_PY)?;
let model_arg = model
.as_ref()
.map(|p| p.display().to_string())
.zip(cfg.model_release.clone());
let extra = text_embed_args(
&input.display().to_string(),
&cfg.text_column,
&extension.display().to_string(),
&out.display().to_string(),
cfg.vector_column.as_deref(),
model_arg
.as_ref()
.map(|(dir, release)| (dir.as_str(), release.as_str())),
);
Ok(uv_run_args(&script.to_string_lossy(), &extra))
}
// ─────────────────────────────────────────────────────────────────────────────
// uv (activate) — a Protocol's OWN Python step, inside the asset graph.
//
// The generic counterpart to the frozen-script operators (`splink_resolve`,
// `gleif_ra_fetch`, `umap_project`, `text_embed`): the same `uv run --script`
// substrate, but the script is the Protocol's rather than one compiled into this
// binary. Without it, a Python step has to be `command:` — which addresses no exact
// bytes, declares no assets, and runs under whatever `python3` the machine has.
//
// TWO PROPERTIES ARE ENFORCED, NOT DOCUMENTED, and they are the whole reason to build
// this rather than ship a nicer `command:`:
//
// * `reads:`/`produces:` are REFUSED when absent, so the step cannot be a graph
// island. The shape this closes is a `command:` step declaring neither: nothing
// downstream knows its output went stale, and the step beside it skips
// `hash_clean` against inputs that moved weeks ago.
// * `sha256:` pins the script's bytes, so `op: uv@1` addresses an EXACT script the
// way `op: text_embed@1` addresses an exact frozen one.
//
// THE COST, ACCEPTED WHEN THIS WAS CUT: an ordinary edit to a `.py` file becomes a
// manifest edit too. That is the trade the frozen-bytes operators already make, and
// here it also does the staleness work — the digest is inside `op_config_hash`, so
// re-pinning is itself what marks the step stale. The refusal carries the digest
// found, so re-pinning is a copy-paste rather than a hunt.
//
// THE SCRIPT IS DELIBERATELY NOT A `reads:` ASSET. Declaring it would be a weaker
// version of the pin — a hash compared against the last run rather than against what
// the manifest says the step is entitled to run — and it would let an edited script
// execute once before anything objected. The pin refuses it instead.
// ─────────────────────────────────────────────────────────────────────────────
struct Uv;
#[derive(Deserialize)]
#[cfg_attr(test, derive(Debug))]
#[serde(deny_unknown_fields)]
struct UvConfig {
/// The Python script to run, relative to the protocol directory.
script: String,
/// The script's SHA-256, lowercase hex. Checked against the bytes on disk before
/// `uv` is spawned; a mismatch is the step's refusal, not a warning.
sha256: String,
/// Third-party requirements, each pinned `package==version`. Passed to `uv run`
/// as `--with`, so the step gets them whether or not the host has them. An
/// unpinned entry is refused at load — see [`unpinned_dependency`].
#[serde(default)]
deps: Vec<String>,
/// Arguments appended after the script path, verbatim and in order.
#[serde(default)]
args: Vec<String>,
/// What the step reads. REQUIRED and non-empty: a Python step invisible to
/// staleness is the defect this operator exists to close.
#[serde(default)]
reads: Vec<String>,
/// What the step produces. REQUIRED and non-empty, for the same reason.
#[serde(default)]
produces: Vec<String>,
}
/// Why `spec` is not a pin, or `None` when it is one.
///
/// The rule is `package==version` and nothing else, and the reason is a fact about
/// time rather than about style: a range resolves to whatever the index holds on the
/// day it runs, so two Runs a month apart install different code while the manifest,
/// the digest and the asset graph all read as unchanged. Every other 508 spelling
/// that leaves the version open is refused by name so the message says which one.
fn unpinned_dependency(spec: &str) -> Option<String> {
let s = spec.trim();
if s.is_empty() {
return Some("it is blank, so it names no package".to_string());
}
if s.contains(';') {
return Some(
"an environment marker (`;`) makes what is installed a property of the \
machine rather than of the manifest"
.to_string(),
);
}
if s.contains('@') {
return Some(
"a direct reference (`@`) names a location, not a version — what is at \
that location can change without the manifest changing"
.to_string(),
);
}
let Some((name, version)) = s.split_once("==") else {
return Some(
"it carries no `==`, so the resolver picks the version on the day it runs".to_string(),
);
};
// `!=` rather than a bare `!`, so a PEP 440 epoch (`1!5.5`) is not refused with a
// message about a second constraint it does not carry. The epoch is still refused
// — by the version character check below, which says so truthfully.
let second_constraint =
|part: &str| part.contains("!=") || part.contains(['<', '>', '~', '=', ',']);
if second_constraint(name) || second_constraint(version) {
return Some(
"a pin is exactly one `==` and nothing else — this carries a second \
constraint, which leaves a range open"
.to_string(),
);
}
let base = match name.split_once('[') {
Some((base, extras)) => {
let Some(extras) = extras.strip_suffix(']') else {
return Some("its `[` is never closed".to_string());
};
if extras.trim().is_empty()
|| !extras
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | ','))
{
return Some(format!("`[{}]` is not a list of extras", extras));
}
base
}
None => name,
};
let base = base.trim();
if base.is_empty()
|| !base
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))
{
return Some(format!("`{}` is not a distribution name", base));
}
let v = version.trim();
if v.is_empty() {
return Some("its `==` has nothing after it".to_string());
}
if v.contains('*') {
return Some(format!(
"`{}` is a wildcard, which is a range written as a pin",
v
));
}
if !v
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+'))
{
return Some(format!("`{}` is not a version", v));
}
None
}
/// The `dependencies` a script declares in its own PEP 723 header, or `Err` when the
/// header is there and cannot be read.
///
/// **THIS IS WHY THE DIGEST PIN IS NOT ENOUGH ON ITS OWN.** `uv run --script` reads a
/// `# /// script … ///` block inside the file and installs what it names, *in addition
/// to* every `--with` on the command line. So a step can declare no `deps:` at all,
/// pin its script byte-for-byte, and still resolve an unpinned package on the day it
/// runs — because the unpinned declaration is *part of the bytes the digest pins*.
/// Reproduced end to end before this existed: a script whose header said
/// `dependencies = ["iniconfig"]` installed one and the Run succeeded.
///
/// The class, which is worth more than the instance: **a byte digest over an artifact
/// secures the artifact's bytes, not what the artifact's own declarative sub-language
/// asks the runtime to fetch at execution time.** A Dockerfile's `FROM` tag inside a
/// pinned Dockerfile is the same shape. Pinning the wrapper is not pinning the
/// behaviour.
///
/// Returning `Err` rather than an empty list when a block cannot be parsed is
/// deliberate: an unreadable declaration is not an absent one, and the whole point of
/// this function is that what it cannot see gets installed anyway.
fn script_declared_dependencies(source: &str) -> std::result::Result<Vec<String>, String> {
// The block is comment lines between `# /// script` and a closing `# ///`, per
// PEP 723. Anchored at line starts so a mention inside the script's own prose is
// not read as the block.
let opener = "# /// script";
let Some(open_at) = source
.find(opener)
.filter(|i| *i == 0 || source.as_bytes()[i - 1] == b'\n')
else {
return Ok(Vec::new());
};
let body_start = open_at + opener.len();
let mut lines = Vec::new();
let mut closed = false;
for line in source[body_start..].lines().skip(1) {
let trimmed = line.trim_end();
if trimmed == "# ///" || trimmed == "#///" {
closed = true;
break;
}
let Some(rest) = trimmed.strip_prefix('#') else {
return Err(format!(
"the PEP 723 block is not closed by a `# ///` line — it runs into \
non-comment source at {:?}",
trimmed.chars().take(40).collect::<String>()
));
};
lines.push(rest.strip_prefix(' ').unwrap_or(rest).to_string());
}
if !closed {
return Err(
"the PEP 723 block opens with `# /// script` and never closes with `# ///`".to_string(),
);
}
let block = lines.join("\n");
// `dependencies = [ … ]`, as a key at the start of a line. Anything else named
// `dependencies` (a value inside another table, a word in a comment) is not it.
let key = "dependencies";
let Some(key_at) = block
.match_indices(key)
.find(|(i, _)| *i == 0 || block.as_bytes()[i - 1] == b'\n')
.map(|(i, _)| i)
else {
return Ok(Vec::new());
};
let after = &block[key_at + key.len()..];
let Some(open_bracket) = after.find('[') else {
return Err("the PEP 723 block declares `dependencies` without a list".to_string());
};
// Reject anything between the key and the bracket that is not `=` and whitespace,
// so `dependencies_note = "…" … [` cannot be read as the list.
if after[..open_bracket]
.chars()
.any(|c| c != '=' && !c.is_whitespace())
{
return Err("the PEP 723 block's `dependencies` is not a list assignment".to_string());
}
let rest = &after[open_bracket + 1..];
let Some(close_bracket) = rest.find(']') else {
return Err("the PEP 723 block's `dependencies` list is not closed".to_string());
};
let mut out = Vec::new();
let mut chars = rest[..close_bracket].chars().peekable();
while let Some(c) = chars.next() {
if c == '"' || c == '\'' {
let quote = c;
let mut value = String::new();
let mut terminated = false;
for c in chars.by_ref() {
if c == quote {
terminated = true;
break;
}
value.push(c);
}
if !terminated {
return Err("the PEP 723 block has an unterminated dependency string".to_string());
}
out.push(value);
}
}
Ok(out)
}
impl UvConfig {
fn parse(with: &Value) -> Result<Self> {
serde_yaml::from_value(with.clone())
.map_err(|e| Error::ManifestValidation(format!("uv: invalid `with:` config: {}", e)))
}
fn validate(&self) -> Result<()> {
if self.script.trim().is_empty() {
return Err(Error::ManifestValidation(
"uv: `script:` is blank — it names the Python file this step runs, \
relative to the protocol directory."
.to_string(),
));
}
// The two declarations that make this an operator rather than a `command:`.
// Absent and empty are refused identically because they cost the same thing:
// a step with no edges is a graph island either way.
for (field, values, what) in [
(
"reads",
&self.reads,
"what it consumes, so a changed input marks it stale",
),
(
"produces",
&self.produces,
"what it writes, so a step downstream of it can go stale in turn",
),
] {
if values.iter().all(|v| v.trim().is_empty()) {
return Err(Error::ManifestValidation(format!(
"uv: `{}:` declares nothing. A `uv@1` step must name {} — that \
declaration is what puts it in the asset graph, and a Python \
step the graph cannot see is the defect this operator exists to \
close. Use `command:` if you genuinely want a step nothing \
tracks.",
field, what
)));
}
}
let d = self.sha256.trim();
if d.len() != 64 || !d.chars().all(|c| c.is_ascii_hexdigit()) {
return Err(Error::ManifestValidation(format!(
"uv: `sha256:` must be 64 hex characters — the SHA-256 of {}. Got \
{} character(s). `shasum -a 256 {}` prints it.",
self.script,
d.len(),
self.script
)));
}
if d.chars().any(|c| c.is_ascii_uppercase()) {
return Err(Error::ManifestValidation(format!(
"uv: `sha256:` is compared as text against a lowercase hex digest, so \
write it lowercase: {}",
d.to_ascii_lowercase()
)));
}
for dep in &self.deps {
if let Some(why) = unpinned_dependency(dep) {
return Err(Error::ManifestValidation(format!(
"uv: dependency `{}` is not pinned — {}. Write it as \
`package==version`, so this Run and a Run next month install the \
same code.",
dep, why
)));
}
}
Ok(())
}
}
/// `uv run --with <dep>… --script <path> <arg>…` — every `--with` before `--script`,
/// in the order the manifest wrote them, so the argv is a function of the config and
/// nothing else. Factored out for the same reason [`text_embed_args`] is: every CI
/// runner here is a machine without `uv`, and this is the half a runner can drive.
fn uv_op_args(script: &str, deps: &[String], args: &[String]) -> Vec<String> {
let mut extra: Vec<String> = Vec::with_capacity(deps.len() * 2);
for dep in deps {
extra.push("--with".to_string());
extra.push(dep.trim().to_string());
}
let mut a = vec!["run".to_string()];
a.extend(extra);
a.push("--script".to_string());
a.push(script.to_string());
a.extend_from_slice(args);
a
}
/// Everything decided BEFORE `uv` is spawned: resolve the script, check its bytes
/// against the pin, build the argv.
///
/// Both refusals are NON-retryable [`Error::StepExecution`], the missing-binary class.
/// A script that is absent will be absent on the second attempt, and one whose bytes
/// disagree with the manifest will disagree again — a Protocol carrying
/// `defaults.retry` would otherwise pay its backoff three times to be told the same
/// thing.
fn uv_invocation(cfg: &UvConfig, dir: &Path) -> Result<Vec<String>> {
let script = dir.join(cfg.script.trim());
let bytes = std::fs::read(&script).map_err(|e| Error::StepExecution {
step: "uv".to_string(),
source: std::io::Error::new(
e.kind(),
format!(
"the script {} is not readable at {}: {}. `uv@1` runs a file the \
Protocol carries; it does not fetch one.",
cfg.script,
script.display(),
e
),
),
})?;
let found = crate::state::content_hash(&bytes);
let expected = cfg.sha256.trim().to_ascii_lowercase();
if found != expected {
return Err(Error::StepExecution {
step: "uv".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"the script {} does not match the digest this step pins.\n \
expected {}\n found {}\nA `uv@1` step addresses exact \
bytes, so editing the script is a manifest edit as well. If the \
new bytes are the ones you want, set `sha256: {}`.",
cfg.script, expected, found, found
),
),
});
}
// EVERY DEPENDENCY REACHING `uv` IS PINNED, WHEREVER IT IS DECLARED. The digest
// above secures the script's bytes; it says nothing about what those bytes ask
// `uv` to fetch. `uv run --script` reads the file's own PEP 723 header and
// installs what it names IN ADDITION TO every `--with`, so a step with no `deps:`
// at all could resolve an unpinned package on the day it ran, with the manifest,
// the digest and the asset graph all reading as unchanged. Same rule, both sites.
let source = String::from_utf8_lossy(&bytes);
let declared = script_declared_dependencies(&source).map_err(|why| Error::StepExecution {
step: "uv".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"{} carries a PEP 723 script block this operator cannot read: {}. \
What it cannot read, `uv` installs anyway, so it is refused rather \
than ignored.",
cfg.script, why
),
),
})?;
for dep in &declared {
if let Some(why) = unpinned_dependency(dep) {
return Err(Error::StepExecution {
step: "uv".to_string(),
source: std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"{}'s own PEP 723 block declares `{}`, which is not pinned — \
{}. `uv run --script` installs what that block names as well \
as what `deps:` names, so pinning the script's bytes does not \
pin what it fetches. Write it as `package==version` there, or \
move it to the step's `deps:`.",
cfg.script, dep, why
),
),
});
}
}
Ok(uv_op_args(&script.to_string_lossy(), &cfg.deps, &cfg.args))
}
impl Operator for Uv {
fn name(&self) -> &'static str {
"uv"
}
fn version(&self) -> semver::Version {
semver::Version::new(1, 0, 0)
}
fn assets(&self, with: &Value) -> Result<OpAssets> {
let cfg = UvConfig::parse(with)?;
cfg.validate()?;
let mut assets = OpAssets::default();
// `default_kind_for_declared_name` and nothing else. These are strings a
// human wrote with no parser behind them — the one declaration site that
// function exists for — so the rule that reads a trailing separator and a
// glob metacharacter is the rule that applies here too.
for name in cfg.reads.iter().map(|n| n.trim()).filter(|n| !n.is_empty()) {
let kind = crate::asset_kind::default_kind_for_declared_name(name);
assets.record_reads(name.to_string(), kind);
}
for name in cfg
.produces
.iter()
.map(|n| n.trim())
.filter(|n| !n.is_empty())
{
let kind = crate::asset_kind::default_kind_for_declared_name(name);
assets.record_produces(name.to_string(), kind);
}
Ok(assets)
}
fn run(&self, with: &Value, ctx: &OpContext) -> Result<StepOutput> {
let cfg = UvConfig::parse(with)?;
cfg.validate()?;
let args = uv_invocation(&cfg, ctx.dir)?;
// Inherit, not Capture: a Protocol's own script is the one place a person
// put print statements on purpose, and Inherit is also the mode that honours
// the step timeout.
run_process("uv", &args, ctx, OutputMode::Inherit, "uv")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn test_ctx<'a>(dir: &'a Path, env: &'a HashMap<String, String>) -> OpContext<'a> {
OpContext {
dir,
db_path: dir,
env,
timeout: None,
cache: None,
}
}
#[test]
fn uv_run_args_arg_order_is_stable() {
assert_eq!(
uv_run_args(
"scripts/x.py",
&["--out".to_string(), "o.parquet".to_string()]
),
vec!["run", "--script", "scripts/x.py", "--out", "o.parquet"]
);
}
// ── datapackage_describe: native-Rust merge + version gate ──────────────────
#[test]
fn parse_finetype_version_extracts_first_dotted_triple() {
assert_eq!(parse_finetype_version("finetype 0.6.53"), Some((0, 6, 53)));
assert_eq!(parse_finetype_version("v12.0.1-rc1"), Some((12, 0, 1)));
}
#[test]
fn parse_finetype_version_none_when_absent() {
assert_eq!(parse_finetype_version("no version here"), None);
}
#[test]
fn require_exact_finetype_version_matching_does_not_raise() {
require_exact_finetype_version("0.6.60", "0.6.60").expect("exact match must pass");
}
#[test]
fn require_exact_finetype_version_newer_than_expected_still_refused() {
// A pin is exact, not a floor — 0.6.61 does not satisfy "expect 0.6.60" even
// though it would satisfy "min 0.6.60".
let err = require_exact_finetype_version("0.6.61", "0.6.60")
.expect_err("a pin is exact, a newer release must still be refused");
let msg = err.to_string();
assert!(msg.contains("0.6.61"));
assert!(msg.contains("0.6.60"));
}
/// The floor is INCLUSIVE ("no older than") — a resolved version EQUAL to
/// MIN_FINETYPE_VERSION must pass, not just anything strictly newer. A `<=`
/// typo in the comparison would refuse exactly this case.
///
/// PURE — no subprocess, no PATH, no tempdir. `require_finetype_at_exact_floor_
/// passes` used to drive this through a REAL spawned `finetype` fake on an
/// isolated PATH; under a heavily loaded machine that spawn was observed to
/// occasionally fail on its own (`StepFailed { code: 1, stderr: "" }`, nothing
/// to do with version comparison), making the test flake for a reason that had
/// nothing to do with what it was pinning. `check_finetype_floor` is the exact
/// same comparison `require_finetype` runs, extracted so this boundary is
/// testable without forking anything.
#[test]
fn check_finetype_floor_at_exact_floor_passes() {
let got = parse_finetype_version(MIN_FINETYPE_VERSION).unwrap();
check_finetype_floor(got, MIN_FINETYPE_VERSION)
.expect("a finetype exactly at the floor must pass, not be refused");
}
#[test]
fn check_finetype_floor_one_patch_below_is_refused() {
let (major, minor, patch) = parse_finetype_version(MIN_FINETYPE_VERSION).unwrap();
let below = (major, minor, patch - 1);
let err = check_finetype_floor(below, MIN_FINETYPE_VERSION)
.expect_err("one patch below the floor must be refused");
assert!(err.to_string().contains("older"));
}
/// The nominations floor replaces the general one for a step that declares
/// `nominations:`, so it has to sit at or above it: below, a step with
/// nominations would admit a binary the general floor exists to refuse.
#[test]
fn the_nominations_floor_is_not_below_the_general_floor() {
assert!(
parse_finetype_version(MIN_FINETYPE_NOMINATIONS_VERSION).unwrap()
>= parse_finetype_version(MIN_FINETYPE_VERSION).unwrap()
);
}
#[test]
fn check_nominations_floor_at_exact_floor_passes() {
let got = parse_finetype_version(MIN_FINETYPE_NOMINATIONS_VERSION).unwrap();
check_nominations_floor(got, MIN_FINETYPE_NOMINATIONS_VERSION)
.expect("the release that carries `--nominations` must pass its own floor");
}
#[test]
fn check_nominations_floor_one_patch_below_is_refused_naming_both_versions() {
let (major, minor, patch) =
parse_finetype_version(MIN_FINETYPE_NOMINATIONS_VERSION).unwrap();
let below = (major, minor, patch - 1);
let msg = check_nominations_floor(below, MIN_FINETYPE_NOMINATIONS_VERSION)
.expect_err("one patch below the nominations release must be refused")
.to_string();
assert!(msg.contains(MIN_FINETYPE_NOMINATIONS_VERSION), "{msg}");
assert!(
msg.contains(&format!("{major}.{minor}.{}", patch - 1)),
"{msg}"
);
assert!(msg.contains("--nominations"), "{msg}");
}
/// PURE, same rationale as `check_finetype_floor_at_exact_floor_passes` above:
/// `resolve_finetype_version` is the exact parse-or-refuse step `require_finetype`
/// runs on `finetype --version`'s stdout, extracted so the unparseable-output
/// refusal is testable without spawning anything.
#[test]
fn resolve_finetype_version_unparseable_fails_closed() {
let err = resolve_finetype_version("finetype (unknown build)").expect_err(
"an unparseable `finetype --version` must stop the run, not default silently",
);
assert!(err.to_string().contains("could not parse"));
}
fn field(name: &str, extra: serde_json::Value) -> serde_json::Value {
let mut obj = extra.as_object().cloned().unwrap_or_default();
obj.insert(
"name".to_string(),
serde_json::Value::String(name.to_string()),
);
serde_json::Value::Object(obj)
}
fn base_descriptor(fields: Vec<serde_json::Value>) -> serde_json::Value {
serde_json::json!({
"name": "widgets",
"resources": [{
"name": "widgets",
"path": "build/widgets.parquet",
"schema": { "fields": fields }
}]
})
}
#[test]
fn merge_datapackage_package_level_overrides_win() {
let base = base_descriptor(vec![field("a", serde_json::json!({}))]);
let overrides = serde_json::json!({ "title": "Widgets — curated" });
let merged = merge_datapackage(base, &overrides).unwrap();
assert_eq!(merged["title"], "Widgets — curated");
// finetype's own package-level key, not mentioned by the override, survives.
assert_eq!(merged["name"], "widgets");
}
#[test]
fn merge_datapackage_field_override_is_a_shallow_replace_not_a_wipe() {
let base = base_descriptor(vec![field(
"cik",
serde_json::json!({ "type": "integer", "x-finetype-label": "identifier" }),
)]);
let overrides = serde_json::json!({
"fields": { "cik": { "description": "the CIK" } }
});
let merged = merge_datapackage(base, &overrides).unwrap();
let f = &merged["resources"][0]["schema"]["fields"][0];
assert_eq!(f["description"], "the CIK");
// Keys the override did not name survive from finetype's half — this is the
// override-precedence rule's OTHER half: overrides win where declared, and
// finetype supplies everything the sidecar does not mention.
assert_eq!(f["type"], "integer");
assert_eq!(f["x-finetype-label"], "identifier");
}
#[test]
fn merge_datapackage_primary_key_referencing_real_column_passes() {
let base = base_descriptor(vec![field("id", serde_json::json!({}))]);
let overrides = serde_json::json!({ "primaryKey": ["id"] });
let merged = merge_datapackage(base, &overrides).unwrap();
assert_eq!(merged["resources"][0]["schema"]["primaryKey"][0], "id");
}
#[test]
fn merge_datapackage_primary_key_referencing_missing_column_hard_fails() {
let base = base_descriptor(vec![field("id", serde_json::json!({}))]);
let overrides = serde_json::json!({ "primaryKey": ["not_a_real_column"] });
let err = merge_datapackage(base, &overrides)
.expect_err("a curated key naming an absent column must be refused, not published");
let msg = err.to_string();
assert!(msg.contains("not_a_real_column"));
assert!(msg.contains("drifted"));
}
#[test]
fn merge_datapackage_foreign_key_referencing_missing_column_hard_fails() {
let base = base_descriptor(vec![field("lei", serde_json::json!({}))]);
let overrides = serde_json::json!({
"foreignKeys": [{ "fields": ["ghost_column"], "reference": { "resource": "gleif", "fields": ["lei"] } }]
});
let err = merge_datapackage(base, &overrides)
.expect_err("a foreignKey naming an absent column must be refused");
assert!(err.to_string().contains("ghost_column"));
}
#[test]
fn merge_datapackage_resource_override_wins() {
// Every production descriptor.overrides.json sets `resource.path` to the
// public URL — finetype writes the local build path, which is never where a
// consumer fetches the resource. This is the resource-level override path,
// distinct from the package-level and field-level ones above it.
let base = base_descriptor(vec![field("id", serde_json::json!({}))]);
let overrides = serde_json::json!({
"resource": { "path": "https://openlake.meridian.online/widgets.parquet" }
});
let merged = merge_datapackage(base, &overrides).unwrap();
assert_eq!(
merged["resources"][0]["path"],
"https://openlake.meridian.online/widgets.parquet"
);
// A resource key the override did not mention survives from finetype's half.
assert_eq!(merged["resources"][0]["name"], "widgets");
}
#[test]
fn merge_datapackage_structural_keys_never_leak_to_the_package_root() {
// Every production overrides file sets `resource` and `primaryKey` at the
// TOP LEVEL of descriptor.overrides.json — these are handled structurally
// (folded into resources[0] / its schema) and must never ALSO land as
// sibling keys on the package root next to title/description/etc.
let base = base_descriptor(vec![field("id", serde_json::json!({}))]);
let overrides = serde_json::json!({
"title": "Widgets",
"resource": { "path": "https://example.com/widgets.parquet" },
"fields": { "id": { "description": "the id" } },
"primaryKey": ["id"]
});
let merged = merge_datapackage(base, &overrides).unwrap();
assert_eq!(merged["title"], "Widgets");
assert!(
merged.get("resource").is_none(),
"`resource` must not leak onto the package root: {merged:#}"
);
assert!(
merged.get("fields").is_none(),
"`fields` must not leak onto the package root: {merged:#}"
);
assert!(
merged.get("primaryKey").is_none(),
"`primaryKey` must not leak onto the package root: {merged:#}"
);
}
/// Pins the serialization shape this operator's byte-identity guarantee rests on: `serde_json`'s
/// pretty printer with NO `preserve_order` in this build's dependency graph
/// (confirmed separately via `cargo tree`) means its `Map` is BTreeMap-backed,
/// so every object's keys serialize in sorted order with no explicit sort step
/// — matching Python's `json.dump(..., indent=2, sort_keys=True)` byte for
/// byte: two-space indent, keys sorted at EVERY nesting level, and an empty
/// container rendered inline (`[]`, not `[\n ]`).
#[test]
fn json_pretty_output_matches_pythons_sort_keys_format() {
let v = serde_json::json!({"b": 1, "a": {"z": 1, "y": 2}, "c": []});
let text = serde_json::to_string_pretty(&v).unwrap();
assert_eq!(
text,
"{\n \"a\": {\n \"y\": 2,\n \"z\": 1\n },\n \"b\": 1,\n \"c\": []\n}"
);
}
#[test]
fn run_process_success_and_capture() {
let dir = std::env::temp_dir();
let env = HashMap::new();
let ctx = test_ctx(&dir, &env);
// Inherit: `true` exits 0 → Ok.
assert!(run_process("true", &[], &ctx, OutputMode::Inherit, "t").is_ok());
// Capture: stdout is returned.
let out = run_process("echo", &["hi".to_string()], &ctx, OutputMode::Capture, "t").unwrap();
assert!(out.stdout.unwrap().contains("hi"));
}
#[test]
fn run_process_nonzero_exit_is_retryable_stepfailed() {
let dir = std::env::temp_dir();
let env = HashMap::new();
let ctx = test_ctx(&dir, &env);
// `false` exits 1 → retryable StepFailed (the engine step-retry re-attempts).
match run_process("false", &[], &ctx, OutputMode::Inherit, "t") {
Err(Error::StepFailed { code, .. }) => assert_eq!(code, 1),
other => panic!("expected StepFailed, got {:?}", other.map(|_| ())),
}
}
#[test]
fn run_process_missing_binary_is_nonretryable_stepexecution() {
let dir = std::env::temp_dir();
let env = HashMap::new();
let ctx = test_ctx(&dir, &env);
// A missing binary is deterministic → NON-retryable StepExecution, so a 4 h
// job is never re-attempted 3× for a typo. This is the load-bearing choice.
match run_process(
"arc-no-such-binary-xyz",
&[],
&ctx,
OutputMode::Inherit,
"t",
) {
Err(Error::StepExecution { .. }) => {}
other => panic!("expected StepExecution, got {:?}", other.map(|_| ())),
}
}
#[test]
fn resolves_by_name_and_semver() {
assert!(resolve("parquet_export").is_ok());
assert!(resolve("parquet_export@1").is_ok());
assert!(resolve("parquet_export@^1.0").is_ok());
assert!(resolve("parquet_export@2").is_err()); // version not satisfied
assert!(resolve("no_such_op").is_err()); // unknown
}
#[test]
fn parquet_export_declares_lineage() {
let with: Value =
serde_yaml::from_str("input: crosswalk_edges\ndest: build/out.parquet").unwrap();
let assets = assets_for("parquet_export", Some(&with)).unwrap();
assert_eq!(assets.reads, vec!["crosswalk_edges".to_string()]);
assert_eq!(assets.produces, vec!["build/out.parquet".to_string()]);
}
#[test]
fn parquet_export_rejects_bad_config() {
// missing required `dest`
let with: Value = serde_yaml::from_str("input: t").unwrap();
assert!(assets_for("parquet_export", Some(&with)).is_err());
// unknown field
let with: Value = serde_yaml::from_str("input: t\ndest: o.parquet\nbogus: 1").unwrap();
assert!(assets_for("parquet_export", Some(&with)).is_err());
}
// ── parquet_export key-value metadata ────────────────────────────────────
//
// The statement tests below build the SQL and never touch a file; the three
// that follow them write real Parquet and compare bytes. Both halves are
// needed: the statement tests say what was asked for, and only the byte tests
// say what landed in the footer.
/// Config helper — parse a `with:` block into the typed config.
fn pq_cfg(yaml: &str) -> ParquetExportConfig {
ParquetExportConfig::parse(&serde_yaml::from_str::<Value>(yaml).unwrap()).unwrap()
}
/// An export declaring no metadata emits the statement this operator
/// built before key-value metadata existed, character for character. Pinned as
/// a literal rather than rebuilt from the config, so a change to the option
/// list cannot quietly agree with itself.
#[test]
fn parquet_export_sql_without_metadata_is_unchanged() {
const UNSTAMPED: &str = "COPY (SELECT * FROM t ORDER BY id) TO '/tmp/o.parquet' \
(FORMAT parquet, COMPRESSION zstd, ROW_GROUP_SIZE 122880);";
// `metadata:` absent entirely.
let absent = pq_cfg(
"input: t\ndest: o.parquet\ncompression: zstd\nrow_group_size: 122880\norder_by: id",
);
assert_eq!(
parquet_export_sql(&absent, Path::new("/tmp/o.parquet")),
UNSTAMPED,
"an export with no `metadata:` must emit the pre-change statement"
);
// `metadata: {}` declared but empty takes the same path — and must, since
// DuckDB rejects `KV_METADATA {}` as a syntax error.
let empty = pq_cfg(
"input: t\ndest: o.parquet\ncompression: zstd\nrow_group_size: 122880\norder_by: id\nmetadata: {}",
);
assert_eq!(
parquet_export_sql(&empty, Path::new("/tmp/o.parquet")),
UNSTAMPED,
"an empty `metadata:` map must not emit a KV_METADATA option"
);
}
/// Declared metadata reaches the statement. This is the test that
/// reddens on a version that accepts the config and drops it.
#[test]
fn parquet_export_sql_stamps_declared_metadata() {
let cfg = pq_cfg("input: t\ndest: o.parquet\nmetadata:\n descriptor: '{\"name\":\"x\"}'");
let sql = parquet_export_sql(&cfg, Path::new("/tmp/o.parquet"));
assert_eq!(
sql,
"COPY (SELECT * FROM t) TO '/tmp/o.parquet' \
(FORMAT parquet, COMPRESSION zstd, KV_METADATA {'descriptor': '{\"name\":\"x\"}'});",
"declared metadata must reach the COPY option list"
);
}
/// A value carrying an apostrophe must be escaped by doubling, or the statement
/// is a syntax error at best and an injection at worst.
#[test]
fn parquet_export_sql_escapes_quotes_in_keys_and_values() {
assert_eq!(sql_string_literal("it's"), "'it''s'");
// Backslash and double-quote are NOT escapes in a DuckDB `'…'` literal, so
// they must pass through untouched — escaping them would corrupt JSON.
assert_eq!(sql_string_literal(r#"{"a":"b\c"}"#), r#"'{"a":"b\c"}'"#);
let cfg = pq_cfg("input: t\ndest: o.parquet\nmetadata:\n \"it's\": \"o'clock\"");
let sql = parquet_export_sql(&cfg, Path::new("/tmp/o.parquet"));
assert!(
sql.contains("KV_METADATA {'it''s': 'o''clock'}"),
"quotes must be doubled in both key and value; got: {sql}"
);
}
/// The entries are emitted in sorted key order however the `with:` block
/// lists them. DuckDB writes the map into the footer in the order given, so an
/// unordered map would move the file's bytes between runs and destroy the
/// reproducibility `order_by` exists to provide.
#[test]
fn parquet_export_metadata_emits_in_sorted_key_order() {
let declared_forwards =
pq_cfg("input: t\ndest: o.parquet\nmetadata:\n aaa: '1'\n zzz: '2'");
let declared_backwards =
pq_cfg("input: t\ndest: o.parquet\nmetadata:\n zzz: '2'\n aaa: '1'");
let forwards = parquet_export_sql(&declared_forwards, Path::new("/tmp/o.parquet"));
let backwards = parquet_export_sql(&declared_backwards, Path::new("/tmp/o.parquet"));
assert_eq!(
forwards, backwards,
"declaration order must not reach the statement"
);
assert!(
forwards.contains("KV_METADATA {'aaa': '1', 'zzz': '2'}"),
"entries must be sorted by key; got: {forwards}"
);
}
/// The generated statement stays legible to SQL introspection: the vendored
/// sqlparser fork parses the open-ended `KV_METADATA` option and its struct
/// literal under the DuckDB dialect. Upstream 0.55.0 rejects the option list
/// outright, so this pins the fork's reach over the new syntax.
#[test]
fn parquet_export_sql_parses_under_the_duckdb_dialect() {
let cfg = pq_cfg(
"input: t\ndest: o.parquet\nrow_group_size: 122880\norder_by: id\nmetadata:\n descriptor: '{\"name\":\"x\"}'",
);
let sql = parquet_export_sql(&cfg, Path::new("/tmp/o.parquet"));
let parsed =
sqlparser::parser::Parser::parse_sql(&sqlparser::dialect::DuckDbDialect {}, &sql);
assert!(
parsed.is_ok(),
"the stamped COPY must parse under the DuckDB dialect, else introspection \
degrades the step to an opaque node; got: {parsed:?}\nsql: {sql}"
);
}
/// A pipeline DB with a two-column table, for the byte-level tests below.
///
/// The rows are **stored out of key order** — `(i * 7919) % 1000` is a
/// permutation of `0..1000`, chosen over `hash(i)` so the scramble is fixed by
/// arithmetic rather than by a DuckDB internal. That is load-bearing, not
/// decoration: a fixture written in key order makes `order_by: id` a no-op, and
/// every claim below about the interaction between stamping and `order_by`
/// would hold vacuously. `parquet_export_stamping_moves_the_hash_but_only_the_footer`
/// asserts the scramble is real before it relies on it.
fn pq_fixture(dir: &Path) -> PathBuf {
let db = dir.join("pipeline.duckdb");
let conn = duckdb::Connection::open(&db).unwrap();
conn.execute_batch(
"CREATE TABLE t AS SELECT i AS id, 'name' || i AS name \
FROM range(1000) AS r(i) ORDER BY (i * 7919) % 1000;",
)
.unwrap();
drop(conn);
db
}
/// Run `parquet_export` with the given `with:` block against `db`, returning the
/// written file's bytes.
fn pq_run(dir: &Path, db: &Path, with_yaml: &str) -> Vec<u8> {
let with: Value = serde_yaml::from_str(with_yaml).unwrap();
let env = HashMap::new();
let ctx = OpContext {
dir,
db_path: db,
env: &env,
timeout: None,
cache: None,
};
ParquetExport.run(&with, &ctx).unwrap();
let dest: String = serde_yaml::from_value::<ParquetExportConfig>(with)
.unwrap()
.dest;
std::fs::read(dir.join(dest)).unwrap()
}
/// The metadata is in the written file's footer, and comes back as the
/// UTF-8 it went in as. `decode()` is the read-back, not a `VARCHAR` cast: the
/// footer map is untyped bytes, so DuckDB hands it over as `BLOB` and casting
/// that to `VARCHAR` yields an escaped rendering rather than the text.
#[test]
fn parquet_export_writes_metadata_readable_off_the_file() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
let db = pq_fixture(dir);
// A value with a quote, a brace and a non-ASCII character — the shapes a
// real descriptor carries.
let descriptor = r#"{"name":"café","note":"it's fine"}"#;
pq_run(
dir,
&db,
&format!(
"input: t\ndest: out/stamped.parquet\norder_by: id\nmetadata:\n descriptor: '{}'\n licence: CC-BY-4.0",
descriptor.replace('\'', "''")
),
);
let conn = duckdb::Connection::open_in_memory().unwrap();
let path = dir.join("out/stamped.parquet");
let mut stmt = conn
.prepare(&format!(
"SELECT decode(key), decode(value) FROM parquet_kv_metadata('{}') ORDER BY 1",
path.display()
))
.unwrap();
let rows: Vec<(String, String)> = stmt
.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))
.unwrap()
.map(|r| r.unwrap())
.collect();
assert_eq!(
rows,
vec![
("descriptor".to_string(), descriptor.to_string()),
("licence".to_string(), "CC-BY-4.0".to_string()),
],
"both entries must be readable off the footer as the UTF-8 that was written"
);
}
/// At the byte level — an export declaring no metadata produces the same
/// bytes as the statement this operator issued before the change. The reference
/// is not a checked-in golden but a COPY run through the same library in the
/// same process: the Parquet footer records the writing DuckDB's version
/// string, so a golden would pin the library, not this operator.
#[test]
fn parquet_export_without_metadata_is_byte_identical_to_the_prior_copy() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
let db = pq_fixture(dir);
// The reference: the exact statement `run` built before `metadata` existed.
let reference_path = dir.join("reference.parquet");
let conn = duckdb::Connection::open(&db).unwrap();
conn.execute_batch(&format!(
"COPY (SELECT * FROM t ORDER BY id) TO '{}' (FORMAT parquet, COMPRESSION zstd, ROW_GROUP_SIZE 122880);",
reference_path.display()
))
.unwrap();
drop(conn);
let reference = std::fs::read(&reference_path).unwrap();
let through_operator = pq_run(
dir,
&db,
"input: t\ndest: unstamped.parquet\ncompression: zstd\nrow_group_size: 122880\norder_by: id",
);
assert_eq!(
through_operator, reference,
"an unstamped export must be byte-identical to the pre-change COPY"
);
// And an explicitly empty map is the same file again.
let empty_map = pq_run(
dir,
&db,
"input: t\ndest: empty.parquet\ncompression: zstd\nrow_group_size: 122880\norder_by: id\nmetadata: {}",
);
assert_eq!(
empty_map, reference,
"an empty `metadata:` map must not move the output bytes"
);
}
/// What stamping costs, measured rather than assumed.
///
/// Three claims, each checked: a stamped export is reproducible (same config,
/// same bytes), stamping changes the file and so its hash, and the change is
/// confined to the footer — every data page is byte-identical, so `order_by`
/// still buys exactly what it bought before. Any publish step that pins the
/// hash of a stamped file has to re-pin it once, not on every run.
#[test]
fn parquet_export_stamping_moves_the_hash_but_only_the_footer() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
let db = pq_fixture(dir);
let unstamped = "input: t\ndest: u.parquet\norder_by: id\nrow_group_size: 122880";
let stamped = "input: t\ndest: s.parquet\norder_by: id\nrow_group_size: 122880\nmetadata:\n descriptor: DESCRIPTOR";
let stamped_again = "input: t\ndest: s2.parquet\norder_by: id\nrow_group_size: 122880\nmetadata:\n descriptor: DESCRIPTOR";
let u = pq_run(dir, &db, unstamped);
let s = pq_run(dir, &db, stamped);
let s2 = pq_run(dir, &db, stamped_again);
// First: `order_by` is load-bearing on this fixture. The rows are stored
// out of key order, so the same export without the clause is a different
// file. Without this the footer-confinement assertion below would pass on a
// fixture where ORDER BY does nothing, and would be measuring nothing.
let unordered = pq_run(
dir,
&db,
"input: t\ndest: unordered.parquet\nrow_group_size: 122880",
);
assert_ne!(
unordered, u,
"the fixture must be stored out of key order, or `order_by` proves nothing here"
);
// Reproducible: the same stamp twice is the same file.
assert_eq!(
s, s2,
"a stamped export must be reproducible — same config, same bytes"
);
// But not the same file as the unstamped one: the hash moves.
assert_ne!(
u, s,
"stamping must change the file, or nothing was written to the footer"
);
// And the change is confined to the footer. Every byte up to the first
// difference is shared, and the difference lands past the last data page —
// located here as the start of the Parquet footer, which is the final
// 8 bytes (4-byte footer length + the `PAR1` magic) plus that length.
let footer_len = u32::from_le_bytes(u[u.len() - 8..u.len() - 4].try_into().unwrap());
let footer_start = u.len() - 8 - footer_len as usize;
let common = u.iter().zip(s.iter()).take_while(|(a, b)| a == b).count();
assert!(
common >= footer_start,
"stamping must not touch the data pages: files diverge at byte {common}, \
but the unstamped footer starts at {footer_start}"
);
assert!(
s.len() > u.len(),
"the stamped file must be the larger one: {} vs {}",
s.len(),
u.len()
);
}
#[test]
fn datapackage_describe_declares_lineage() {
let with: Value = serde_yaml::from_str(
"parquet: build/edgar_gleif.parquet\noverrides: descriptor.overrides.json\nout: datapackage.json",
)
.unwrap();
let a = assets_for("datapackage_describe", Some(&with)).unwrap();
// The curated sidecar is a read: an edit to it must re-describe.
assert_eq!(
a.reads,
vec![
"build/edgar_gleif.parquet".to_string(),
"descriptor.overrides.json".to_string()
]
);
assert_eq!(a.produces, vec!["datapackage.json".to_string()]);
// So is the nominations file, when the step declares one.
let with_nominations: Value = serde_yaml::from_str(
"parquet: p\noverrides: o\nout: d\nnominations: nominations.finetype.json",
)
.unwrap();
let a = assets_for("datapackage_describe", Some(&with_nominations)).unwrap();
assert_eq!(
a.reads,
vec![
"p".to_string(),
"o".to_string(),
"nominations.finetype.json".to_string()
]
);
// unknown field rejected (deny_unknown_fields)
let bad: Value = serde_yaml::from_str("parquet: p\noverrides: o\nout: d\nx: 1").unwrap();
assert!(assets_for("datapackage_describe", Some(&bad)).is_err());
}
#[test]
fn splink_resolve_declares_lineage() {
// reads = [edgar, gleif] (order-preserving), produces = [out].
let with: Value = serde_yaml::from_str(
"edgar: build/sec_entities.parquet\ngleif: build/gleif.parquet\nout: build/resolved.parquet",
)
.unwrap();
let a = assets_for("splink_resolve", Some(&with)).unwrap();
assert_eq!(
a.reads,
vec![
"build/sec_entities.parquet".to_string(),
"build/gleif.parquet".to_string()
]
);
assert_eq!(a.produces, vec!["build/resolved.parquet".to_string()]);
// `sample` is optional — a published run omits it and still validates.
assert!(assets_for("splink_resolve", Some(&with)).is_ok());
// unknown field rejected (deny_unknown_fields)
let bad: Value = serde_yaml::from_str("edgar: e\ngleif: g\nout: o\nbogus: 1").unwrap();
assert!(assets_for("splink_resolve", Some(&bad)).is_err());
// missing a required field rejected
let miss: Value = serde_yaml::from_str("edgar: e\ngleif: g").unwrap();
assert!(assets_for("splink_resolve", Some(&miss)).is_err());
}
#[test]
fn splink_resolve_resolves_by_semver() {
assert!(resolve("splink_resolve").is_ok());
assert!(resolve("splink_resolve@1").is_ok());
assert!(resolve("splink_resolve@^1.0").is_ok());
assert!(resolve("splink_resolve@2").is_err()); // version not satisfied
}
#[test]
fn splink_resolve_sample_omitted_for_published_run() {
// Published run: no sample → the full corpus, no `--sample` flag at all
// (it MUST be omittable — a `--sample 0` would cap GLEIF to zero rows).
let full = splink_resolve_args("e.parquet", "g.parquet", "o.parquet", None);
assert_eq!(
full,
vec![
"--edgar",
"e.parquet",
"--gleif",
"g.parquet",
"--out",
"o.parquet"
]
);
assert!(!full.iter().any(|a| a == "--sample"));
// `sample: 0` is treated as "full corpus" too → still omitted.
let zero = splink_resolve_args("e.parquet", "g.parquet", "o.parquet", Some(0));
assert!(!zero.iter().any(|a| a == "--sample"));
// A positive smoke-test cap IS emitted, as the trailing `--sample N` pair.
let sampled = splink_resolve_args("e.parquet", "g.parquet", "o.parquet", Some(20_000));
assert_eq!(
&sampled[sampled.len() - 2..],
&["--sample".to_string(), "20000".to_string()]
);
// Full invocation threads through uv_run_args in the documented order.
let script = "/tmp/arcform-op-splink_resolve-1.0.0/splink_resolve.py";
let argv = uv_run_args(script, &sampled);
assert_eq!(argv[0], "run");
assert_eq!(argv[1], "--script");
assert_eq!(argv[2], script);
assert_eq!(argv[3], "--edgar");
}
#[test]
fn gleif_ra_fetch_is_in_catalog_and_versioned() {
assert!(resolve("gleif_ra_fetch").is_ok());
assert!(resolve("gleif_ra_fetch@1").is_ok());
assert!(resolve("gleif_ra_fetch@^1.0").is_ok());
assert!(resolve("gleif_ra_fetch@2").is_err()); // version not satisfied
}
#[test]
fn gleif_ra_fetch_declares_only_its_output() {
// Ingress: produces the local CSV, reads no graph node (the GLEIF API is not
// in the AssetGraph). `out` is lowercased into the produces asset.
let with: Value =
serde_yaml::from_str("ra: RA000665\nout: build/gleif_ra_sec.csv").unwrap();
let a = assets_for("gleif_ra_fetch", Some(&with)).unwrap();
assert_eq!(a.produces, vec!["build/gleif_ra_sec.csv".to_string()]);
assert!(a.reads.is_empty());
}
#[test]
fn gleif_ra_fetch_optional_fields_default() {
// page_size / user_agent are optional — a minimal ra+out config validates.
let min: Value = serde_yaml::from_str("ra: RA000665\nout: g.csv").unwrap();
assert!(assets_for("gleif_ra_fetch", Some(&min)).is_ok());
// …and both may be supplied.
let full: Value =
serde_yaml::from_str("ra: RA000665\nout: g.csv\npage_size: 50\nuser_agent: 'X (y@z)'")
.unwrap();
assert!(assets_for("gleif_ra_fetch", Some(&full)).is_ok());
}
#[test]
fn gleif_ra_fetch_rejects_bad_config() {
// missing required `out`
let no_out: Value = serde_yaml::from_str("ra: RA000665").unwrap();
assert!(assets_for("gleif_ra_fetch", Some(&no_out)).is_err());
// missing required `ra`
let no_ra: Value = serde_yaml::from_str("out: g.csv").unwrap();
assert!(assets_for("gleif_ra_fetch", Some(&no_ra)).is_err());
// unknown field rejected (deny_unknown_fields) — e.g. a stray `max_pages`,
// which is a smoke-test-only CLI flag and deliberately not a config field.
let bogus: Value = serde_yaml::from_str("ra: RA000665\nout: g.csv\nmax_pages: 2").unwrap();
assert!(assets_for("gleif_ra_fetch", Some(&bogus)).is_err());
}
// ── umap_project ─────────────────────────────────────────────────────────
//
// The catalog entry is UNCONDITIONAL, beside the other uv-run operators and
// unlike the ureq-backed ones. arcform validates a manifest against the same
// catalog it executes from, so a feature gate named for the transport would
// force a consumer that only wants to READ a Protocol naming this step — the
// brightfield viewer — to claim the capability to RUN it.
// -----------------------------------------------------------------------
// What the sidecar guard ASSEMBLES, asserted as a value.
//
// The end-to-end tests in tests/datapackage_describe.rs pin what the guard
// DECIDES — refuse, warn, stay silent — by searching the real `arc` binary's
// stderr. `contains` is monotone: adding a line, or swapping one rendering for
// another, leaves every substring it was asked about still present. So three
// branches whose only effect is on message content shipped unpinned, and the
// mutation gate found all three. These compare the whole string, which is not
// monotone, and the degenerate inputs those branches exist for are driven here
// because reaching them through a real `arc run` needs a base descriptor no
// pipeline would produce.
// -----------------------------------------------------------------------
/// Call the guard with a `fields` override block and an explicit map of what the
/// engine nominated, so a test states both halves of the comparison outright.
fn guard(
overrides: serde_json::Value,
engine_nominated: &[(&str, serde_json::Value)],
) -> Result<Vec<String>> {
let nominated: std::collections::BTreeMap<String, serde_json::Value> = engine_nominated
.iter()
.map(|(name, constraints)| ((*name).to_string(), constraints.clone()))
.collect();
check_field_claims(overrides.as_object().expect("a JSON object"), &nominated)
}
/// The refusal text a maintainer actually reads, or a panic naming what came back
/// instead — a guard that returned the wrong error VARIANT would otherwise read as
/// a passing assertion on a message nobody sees.
fn refusal(result: Result<Vec<String>>) -> String {
match result {
Err(Error::StepFailed { stderr, .. }) => stderr,
Err(other) => panic!("expected a StepFailed, got {other}"),
Ok(warnings) => panic!("expected a refusal, got warnings: {warnings:?}"),
}
}
const REFUSAL_HEADER: &str = "datapackage_describe: descriptor.overrides.json \
states a type claim that is not its to state — a declared type, its label and \
its confidence belong in nominations.finetype.json, which finetype reads, not \
in the curated sidecar, which nothing reads back.";
#[test]
fn a_pure_forgery_refusal_carries_no_contradiction_section() {
let text = refusal(guard(
serde_json::json!({"note": {"x-finetype-nominated": true}}),
&[],
));
assert_eq!(
text,
format!(
"{REFUSAL_HEADER}\n forges the nomination mark: 'note' → 'x-finetype-nominated'"
)
);
}
#[test]
fn a_pure_contradiction_refusal_carries_no_forgery_section() {
// The section headers are emitted only for the arm that fired. A refusal that
// printed an empty "forges the nomination mark:" line here would accuse the
// maintainer of forging nothing, on an input that forged nothing — and every
// substring the end-to-end tests look for would still be present.
let text = refusal(guard(
serde_json::json!({"id": {"type": "integer"}}),
&[("id", serde_json::json!({"minLength": 4}))],
));
assert_eq!(
text,
format!(
"{REFUSAL_HEADER}\n contradicts a field finetype already nominated: 'id' → 'type'"
)
);
}
#[test]
fn a_refusal_that_fires_both_arms_carries_both_sections_in_order() {
let text = refusal(guard(
serde_json::json!({
"note": {"x-finetype-nominated": true},
"id": {"type": "integer", "x-finetype-label": "identifier"},
}),
&[("id", serde_json::json!({"minLength": 4}))],
));
assert_eq!(
text,
format!(
"{REFUSAL_HEADER}\n \
forges the nomination mark: 'note' → 'x-finetype-nominated'\n \
contradicts a field finetype already nominated: \
'id' → 'type', 'id' → 'x-finetype-label'"
)
);
}
#[test]
fn the_pre_emption_warning_is_one_exact_line_per_claimed_key() {
let warnings = guard(
serde_json::json!({"note": {"type": "integer", "x-finetype-confidence": 0.5}}),
&[],
)
.expect("a claim on a field nobody nominated warns rather than refusing");
assert_eq!(
warnings,
vec![
"datapackage_describe: WARNING field 'note' sets 'type' in \
descriptor.overrides.json, pre-empting finetype's typing. Declare the \
type in nominations.finetype.json instead — this override will be \
refused once the published sidecars are clean."
.to_string(),
"datapackage_describe: WARNING field 'note' sets 'x-finetype-confidence' in \
descriptor.overrides.json, pre-empting finetype's typing. Declare the \
type in nominations.finetype.json instead — this override will be \
refused once the published sidecars are clean."
.to_string(),
]
);
}
#[test]
fn the_lost_bounds_warning_prints_both_constraint_sets() {
let warnings = guard(
serde_json::json!({"id": {"constraints": {"minLength": 8}}}),
&[("id", serde_json::json!({"minLength": 4, "maxLength": 16}))],
)
.expect("replacing a nominated field's constraints warns rather than refusing");
assert_eq!(
warnings,
vec![
"datapackage_describe: WARNING field 'id' is nominated and \
descriptor.overrides.json replaces its constraints — the bounds that came \
with the nomination in nominations.finetype.json are gone unless the \
sidecar restates them.\n \
nominated: {\"maxLength\":16,\"minLength\":4}\n \
sidecar: {\"minLength\":8}"
.to_string(),
]
);
}
#[test]
fn a_nomination_that_carried_no_bounds_renders_as_none_not_null() {
// The case `render_constraints` exists for, and an ordinary one: a taxonomy
// definition with no validation bounds nominates a column with no
// `constraints` at all. Rendering that as the bare `null` a JSON dump would
// produce reads as a value somebody set. Nothing drove this path before —
// every fixture nominated a field WITH bounds — so the branch was unpinned,
// and a substring assertion could not have pinned it in either case.
let warnings = guard(
serde_json::json!({"id": {"constraints": {"minLength": 8}}}),
&[("id", serde_json::Value::Null)],
)
.expect("a nomination with no bounds still warns rather than refusing");
assert_eq!(
warnings,
vec![
"datapackage_describe: WARNING field 'id' is nominated and \
descriptor.overrides.json replaces its constraints — the bounds that came \
with the nomination in nominations.finetype.json are gone unless the \
sidecar restates them.\n \
nominated: (none)\n \
sidecar: {\"minLength\":8}"
.to_string(),
]
);
}
#[test]
fn a_field_block_that_is_not_an_object_claims_nothing() {
// `"note": "a string"` is malformed rather than hostile — the merge ignores it
// (only `as_object()` blocks are copied), so the guard must ignore it too
// rather than refusing a sidecar the merge would have tolerated.
let warnings = guard(
serde_json::json!({"note": "not an object", "id": ["nor", "this"]}),
&[("id", serde_json::json!({"minLength": 4}))],
)
.expect("a non-object field block is ignored, not refused");
assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");
}
#[test]
fn a_base_field_with_no_name_is_not_a_column_and_carries_no_nomination() {
// A nomination is addressed by column name. finetype's own output always names
// its fields, so this base is malformed — but the map that records what the
// engine nominated has to DROP a nameless field rather than filing it under the
// empty string, or a sidecar block keyed `""` inherits that field's nomination
// and every type claim in it is refused. Driven through `merge_datapackage`
// itself because that is where the map is built.
let base = serde_json::json!({
"resources": [{"schema": {"fields": [
{"x-finetype-nominated": true, "constraints": {"minLength": 4}},
{"name": "note", "type": "string"},
]}}]
});
let overrides = serde_json::json!({"fields": {"": {"type": "integer"}}});
let merged = merge_datapackage(base, &overrides)
.expect("a nameless base field must not lend its nomination to the '' block");
let fields = merged["resources"][0]["schema"]["fields"]
.as_array()
.expect("fields survive the merge");
// The nameless field is skipped by the per-field loop too, so the `""` block
// reaches nothing: no `type` was applied to either field.
assert_eq!(fields[0].get("type"), None);
assert_eq!(fields[1]["type"], "string");
}
#[test]
fn umap_project_is_in_catalog_and_versioned() {
let op = resolve("umap_project@1").expect("umap_project is in the catalog");
assert_eq!(op.name(), "umap_project");
assert_eq!(op.version(), semver::Version::new(1, 2, 0));
// No feature gate: a consumer built without `http-fetch` still resolves it.
assert!(
catalog().iter().any(|o| o.name() == "umap_project"),
"umap_project must be in the catalog unconditionally"
);
}
/// 1.1.0 added `nominations:`. `@1` must still resolve, which is what keeps the
/// four published Protocols and every manifest string in the tests loading.
#[test]
fn datapackage_describe_is_in_catalog_and_versioned() {
let op = resolve("datapackage_describe@1").expect("datapackage_describe@1 resolves");
assert_eq!(op.name(), "datapackage_describe");
assert_eq!(op.version(), semver::Version::new(1, 1, 0));
}
#[test]
fn text_embed_is_in_catalog_and_versioned() {
let op = resolve("text_embed@1").expect("text_embed is in the catalog");
assert_eq!(op.name(), "text_embed");
assert_eq!(op.version(), semver::Version::new(1, 0, 0));
assert!(
catalog().iter().any(|o| o.name() == "text_embed"),
"text_embed must be in the catalog unconditionally"
);
}
/// The merged name is GONE, not aliased. It named two jobs, so a manifest asking
/// for it cannot be served correctly by either half — and an alias that quietly
/// resolved to one of them would hand a Protocol a step it did not ask for. The
/// refusal is the operator-unknown one, which names the catalog.
#[test]
fn the_name_that_covered_two_jobs_resolves_to_nothing() {
for reference in ["embed_project", "embed_project@1"] {
// `expect_err` needs `Debug` on the Ok side and `&dyn Operator` has none,
// so the success case is spelled out — which also lets it report WHICH
// operator answered to the merged name.
let err = match resolve(reference) {
Ok(op) => panic!(
"`{reference}` named two jobs and must no longer resolve, but it \
resolved to `{}`",
op.name()
),
Err(e) => e,
};
assert!(
err.to_string().contains("unknown operator 'embed_project'"),
"the refusal names the operator that is gone: {err}"
);
}
assert!(
!catalog().iter().any(|o| o.name() == "embed_project"),
"no catalog entry may still carry the merged name"
);
}
/// The load-time half of the no-model guarantee, and the half CI can run: the projection declares an
/// input and an output and NOTHING ELSE. No model asset appears in the graph
/// because none is needed — a step that placed numbers on a map by way of a model
/// would still be two jobs.
#[test]
fn umap_project_declares_one_input_and_one_output_and_no_model() {
let with: Value = serde_yaml::from_str(
"input: build/homes.parquet\ncolumns: [longitude, latitude]\nout: build/mapped.parquet",
)
.unwrap();
let assets = assets_for("umap_project", Some(&with)).unwrap();
assert_eq!(assets.reads, vec!["build/homes.parquet"]);
assert_eq!(assets.produces, vec!["build/mapped.parquet"]);
assert_eq!(
assets.kinds.get("build/homes.parquet"),
Some(&crate::asset_kind::AssetKind::File)
);
assert_eq!(
assets.kinds.get("build/mapped.parquet"),
Some(&crate::asset_kind::AssetKind::File)
);
assert!(
!assets
.kinds
.values()
.any(|k| *k == crate::asset_kind::AssetKind::Directory),
"the projection reads no model directory — it takes numbers that already \
exist: {:?}",
assets.kinds
);
}
/// The same no-model guarantee at the config level. `deny_unknown_fields` is what makes this a refusal
/// rather than a field silently ignored: a manifest carried over from the merged
/// operator stops at load, naming the field, instead of quietly projecting
/// without embedding anything.
#[test]
fn umap_project_refuses_the_fields_that_belonged_to_the_other_job() {
let base = "input: i.parquet\ncolumns: [lon, lat]\nout: o.parquet\n";
for orphan in ["text_column: description", "model: models/potion"] {
let with: Value = serde_yaml::from_str(&format!("{base}{orphan}")).unwrap();
let err = assets_for("umap_project", Some(&with))
.expect_err(&format!("`{orphan}` is meaningless to a projection"));
let field = orphan.split(':').next().unwrap();
assert!(
err.to_string().contains(field),
"the refusal names the field that does not belong: {err}"
);
}
}
/// The embedding step carries no projection knob, for the same reason: a field
/// that cannot change what the step produces must not be settable on it.
#[test]
fn text_embed_refuses_the_fields_that_belonged_to_the_other_job() {
let base = "input: i.parquet\ntext_column: t\nextension: e\nout: o.parquet\n";
for orphan in ["neighbors: 15", "min_dist: 0.1", "metric: cosine"] {
let with: Value = serde_yaml::from_str(&format!("{base}{orphan}")).unwrap();
let err = assets_for("text_embed", Some(&with))
.expect_err(&format!("`{orphan}` is meaningless to an embedding"));
let field = orphan.split(':').next().unwrap();
assert!(
err.to_string().contains(field),
"the refusal names the field that does not belong: {err}"
);
}
}
#[test]
fn umap_project_rejects_bad_config() {
for missing in [
"columns: [lon]\nout: o",
"input: i\nout: o",
"input: i\ncolumns: [lon]",
] {
let with: Value = serde_yaml::from_str(missing).unwrap();
assert!(
assets_for("umap_project", Some(&with)).is_err(),
"a `with:` block missing a required field must not load: {missing}"
);
}
let bogus: Value =
serde_yaml::from_str("input: i\ncolumns: [lon]\nout: o\nbogus: 1").unwrap();
assert!(assets_for("umap_project", Some(&bogus)).is_err());
// `columns:` is a list of names, not one name.
let scalar: Value = serde_yaml::from_str("input: i\ncolumns: lon\nout: o").unwrap();
assert!(
assets_for("umap_project", Some(&scalar)).is_err(),
"`columns:` takes a sequence — a bare string must not load"
);
}
/// Everything decidable from the manifest alone is refused at load rather than an
/// hour into a run.
#[test]
fn umap_project_refuses_a_setting_that_cannot_make_a_map() {
let base = "input: i\nout: o\n";
for (extra, expected) in [
("columns: []", "is empty"),
("columns: [lon, lat, lon]", "more than once"),
("columns: [lon]\nneighbors: 1", "below 2"),
("columns: [lon]\nneighbors: 0", "below 2"),
("columns: [lon]\nmin_dist: 1.0", "outside [0, 1)"),
("columns: [lon]\nmin_dist: -0.1", "outside [0, 1)"),
("columns: [lon]\nmetric: manhattan", "euclidean or cosine"),
] {
let with: Value = serde_yaml::from_str(&format!("{base}{extra}")).unwrap();
let err = assets_for("umap_project", Some(&with))
.expect_err(&format!("`{extra}` must be refused at load"));
assert!(
err.to_string().contains(expected),
"`{extra}` must be refused naming the bound, got: {err}"
);
}
// The settings that ARE usable must still load.
for extra in [
"columns: [lon]",
"columns: [lon, lat]",
"columns: [lon]\nneighbors: 2",
"columns: [lon]\nmin_dist: 0.0",
"columns: [lon]\nmin_dist: 0.999",
"columns: [lon]\nmetric: euclidean",
"columns: [lon]\nmetric: cosine",
] {
let with: Value = serde_yaml::from_str(&format!("{base}{extra}")).unwrap();
assert!(
assets_for("umap_project", Some(&with)).is_ok(),
"`{extra}` is a usable setting and must load"
);
}
}
/// Unset knobs are OMITTED from the argv entirely, so the frozen script's own
/// defaults stay the single place they are written down.
#[test]
fn umap_project_args_omit_the_knobs_the_manifest_did_not_set() {
assert_eq!(
umap_project_args(
"h.parquet",
&["longitude".to_string(), "latitude".to_string()],
"o.parquet",
None,
None,
None,
None,
),
vec![
"--input",
"h.parquet",
"--column",
"longitude",
"--column",
"latitude",
"--out",
"o.parquet",
],
);
}
#[test]
fn umap_project_args_pass_every_column_and_knob_through() {
assert_eq!(
umap_project_args(
"h.parquet",
&[
"longitude".to_string(),
"latitude".to_string(),
"median_income".to_string(),
],
"o.parquet",
Some("homes.fit"),
Some(30),
Some(0.25),
Some("cosine"),
),
vec![
"--input",
"h.parquet",
"--column",
"longitude",
"--column",
"latitude",
"--column",
"median_income",
"--out",
"o.parquet",
"--fit",
"homes.fit",
"--neighbors",
"30",
"--min-dist",
"0.25",
"--metric",
"cosine",
],
);
}
/// What is handed to `uv`, decided without spawning it: the frozen script is
/// materialised, it is named first, and the bytes on disk are the bytes `@1`
/// addresses. This is the half of `run` a machine with no `uv` can still check —
/// and every CI runner here is one.
///
/// All three optional knobs are set here, to distinct non-default values, rather
/// than left unset as an argv-builder test alone would: `umap_project_args_*`
/// already proves the pure builder emits the right flags when handed values
/// directly, but nothing before this test drove the WIRING between a parsed
/// config and that builder — `cfg.neighbors`, `cfg.min_dist` and
/// `cfg.metric.as_deref()` at the call site in `umap_project_invocation`. A config
/// with every knob unset cannot tell "threaded" from "silently dropped", because
/// omitted-and-defaulted look identical in the argv: setting all three here is
/// what makes this test redden if any one of the three is replaced with `None` or
/// a constant at that call site.
#[test]
fn umap_project_invocation_materialises_the_frozen_script_and_names_it() {
let tmp = tempfile::tempdir().unwrap();
let with: Value = serde_yaml::from_str(
"input: build/homes.parquet\ncolumns: [longitude, latitude]\nout: build/mapped.parquet\nneighbors: 30\nmin_dist: 0.25\nmetric: cosine",
)
.unwrap();
let cfg = UmapProjectConfig::parse(&with).unwrap();
let args = umap_project_invocation(&cfg, tmp.path()).expect("a projection proceeds");
assert_eq!(args[0], "run");
assert_eq!(args[1], "--script");
let script = std::path::Path::new(&args[2]);
assert_eq!(
std::fs::read_to_string(script).unwrap(),
UMAP_PROJECT_PY,
"`op@1` addresses the embedded bytes: what lands on disk is what is \
compiled in, not a script found by path"
);
assert_eq!(
&args[3..],
[
"--input",
&tmp.path().join("build/homes.parquet").display().to_string(),
"--column",
"longitude",
"--column",
"latitude",
"--out",
&tmp.path()
.join("build/mapped.parquet")
.display()
.to_string(),
"--neighbors",
"30",
"--min-dist",
"0.25",
"--metric",
"cosine",
],
"every path reaches the script resolved against the protocol directory, \
and `neighbors:`, `min_dist:` and `metric:` reach the invocation with the \
values the manifest set — not just the values `umap_project_args` would \
emit if handed them directly"
);
assert!(
tmp.path().join("build").is_dir(),
"the output's parent directory is created before the script is asked to \
write into it"
);
}
/// The load-time half of the persisted fit, and the half CI can run: a Protocol
/// that sets `fit:` puts the fitted projection in the asset graph on BOTH sides of
/// the step.
///
/// Asserting membership is not enough on its own and the shape of this test says
/// why. `reads` alone would declare a dependency on a file nothing is answerable
/// for; `produces` alone would hash the bytes without saying the step consumes
/// them. So both lists are checked for the name, the `out` and `input` entries
/// are checked to still be exactly where they were (a fit that displaced the
/// output would pass a looser `contains`), and the kind is checked — a fit
/// declared as anything but a File would be hashed by the wrong branch of
/// `produced_artifact_hash` and a missing one would read as unchanged.
#[test]
fn umap_project_declares_the_fit_it_reads_and_the_fit_it_writes() {
let with: Value = serde_yaml::from_str(
"input: build/homes.parquet\ncolumns: [longitude, latitude]\n\
out: build/mapped.parquet\nfit: build/homes.fit",
)
.unwrap();
let assets = assets_for("umap_project", Some(&with)).unwrap();
assert_eq!(
assets.reads,
vec!["build/homes.parquet", "build/homes.fit"],
"a persisted fit is something this step CONSUMES — a second run reads it \
rather than refitting"
);
assert_eq!(
assets.produces,
vec!["build/mapped.parquet", "build/homes.fit"],
"and something it WRITES — the first run creates it, and deleting it has \
to mark the step stale rather than reading as a clean run"
);
assert_eq!(
assets.kinds.get("build/homes.fit"),
Some(&crate::asset_kind::AssetKind::File),
"the fit is one file: a Directory or Pattern kind would be hashed by a \
branch that cannot see its bytes"
);
}
/// The control for the test above, and the compatibility claim the version bump
/// rests on: a Protocol that sets no `fit:` declares exactly what it declared
/// before the field existed. Without this, "the fit is in the graph" could be
/// satisfied by a graph that always carries one, which would put a file no
/// Protocol asked for into every projection's artifact hash and force a permanent
/// refit.
#[test]
fn umap_project_without_a_fit_declares_no_extra_asset() {
let with: Value = serde_yaml::from_str(
"input: build/homes.parquet\ncolumns: [longitude, latitude]\n\
out: build/mapped.parquet",
)
.unwrap();
let assets = assets_for("umap_project", Some(&with)).unwrap();
assert_eq!(assets.reads, vec!["build/homes.parquet"]);
assert_eq!(assets.produces, vec!["build/mapped.parquet"]);
assert_eq!(
assets.kinds.len(),
2,
"no `fit:` is no third asset: {:?}",
assets.kinds
);
}
/// The other end of the same wire: `fit:` reaches the script's argv, resolved
/// against the protocol directory exactly as `input:` and `out:` are, and its
/// parent directory exists before the script is asked to write into it.
///
/// A separate directory from the output's on purpose. With both under `build/`,
/// the `create_dir_all` for `out` would make the assertion below pass whether or
/// not the fit's own parent was ever created — which is the case that bites a
/// Protocol keeping its fits somewhere other than beside its Parquet, and the
/// exact shape of a test that is green because something else did the work.
#[test]
fn umap_project_invocation_passes_the_fit_resolved_against_the_protocol_dir() {
let tmp = tempfile::tempdir().unwrap();
let with: Value = serde_yaml::from_str(
"input: build/homes.parquet\ncolumns: [longitude, latitude]\n\
out: build/mapped.parquet\nfit: fits/homes.fit",
)
.unwrap();
let cfg = UmapProjectConfig::parse(&with).unwrap();
let args = umap_project_invocation(&cfg, tmp.path()).expect("a projection proceeds");
let fit = tmp.path().join("fits/homes.fit").display().to_string();
let at = args
.iter()
.position(|a| a == "--fit")
.expect("a manifest that sets `fit:` reaches the script with --fit");
assert_eq!(
args[at + 1],
fit,
"the fit is resolved against the protocol directory, not passed through \
as the manifest spelled it — a relative path would land wherever `uv` \
happened to be started from"
);
assert!(
tmp.path().join("fits").is_dir(),
"the fit's own parent directory is created before the script is asked to \
write into it, even when it is not the output's"
);
}
/// The frozen-script path and the operator's own `version()` agree.
///
/// Three literals say what version this operator is — `version()`, the string
/// `umap_project_invocation` hands `materialize_frozen_script`, and the test-only
/// [`UMAP_PROJECT_CACHE_DIR`] — and nothing but this test stops them drifting. The
/// cost of drift is quiet: a bump applied to `version()` alone would leave two
/// releases of the script sharing one cache directory, which is precisely what
/// `op@<version>` addressing exact bytes exists to prevent.
#[test]
fn umap_project_materialises_its_script_under_its_own_version() {
let tmp = tempfile::tempdir().unwrap();
let with: Value =
serde_yaml::from_str("input: i.parquet\ncolumns: [longitude]\nout: o.parquet").unwrap();
let cfg = UmapProjectConfig::parse(&with).unwrap();
let args = umap_project_invocation(&cfg, tmp.path()).expect("a projection proceeds");
let version = resolve("umap_project@1").unwrap().version().to_string();
let expected = format!("arcform-op-umap_project-{version}");
assert_eq!(
expected, UMAP_PROJECT_CACHE_DIR,
"the cache-directory trap has to name the version the operator reports, \
or the tests that drive the cache stop biting the real path"
);
assert!(
std::path::Path::new(&args[2])
.parent()
.unwrap()
.ends_with(&expected),
"the script materialises under the operator's own version; got {}",
args[2]
);
}
/// A setting outside its bounds stops the step, not just the manifest load. `run`
/// re-validates rather than trusting that `assets` was called first — nothing in
/// the trait says it was.
#[test]
fn umap_project_run_refuses_an_out_of_range_knob() {
let tmp = tempfile::tempdir().unwrap();
let env = HashMap::new();
let ctx = test_ctx(tmp.path(), &env);
let with: Value = serde_yaml::from_str(
"input: homes.parquet\ncolumns: [longitude]\nout: out.parquet\nneighbors: 1",
)
.unwrap();
let err = UmapProject
.run(&with, &ctx)
.expect_err("`neighbors: 1` cannot make a map and must stop the step");
assert!(
err.to_string().contains("below 2"),
"the refusal names the bound: {err}"
);
}
/// The script cache is where `?` earns its keep, and both of its failure paths are
/// driven here for real rather than reasoned about — for BOTH uv-run operators
/// this change adds, through `Operator::run` rather than through the invocation
/// helper directly.
///
/// `materialize_frozen_script` fails two ways — the cache directory cannot be
/// created, or the script cannot be written inside it — and a `?` that folded
/// either into `PathBuf::default()` would hand `uv` an empty `--script` path.
/// That is not a quieter error: measured on 2026-08-24, `uv run --script ""` with
/// `current_dir` set to the Protocol directory runs `<protocol>/__main__.py` if one
/// exists and EXITS 0 — a failed step reported as a success, executing whatever is
/// in the Protocol directory, with no output written.
///
/// THROUGH `run`, and per operator, because that is what the coverage is about.
/// The helper is shared, so one test over one operator's invocation function looks
/// like enough; it is not. Each operator has its own `?` on the helper and its own
/// `?` on the invocation call inside `run`, and the mutation gate rewrote three of
/// those four lines with the suite still green when this test drove only
/// `umap_project_invocation` directly. Reaching them means entering at `run`.
///
/// IN A CHILD PROCESS, and that is the whole reason this test looks the way it
/// does. The cache path comes from `std::env::temp_dir()`, which reads `TMPDIR`;
/// `TMPDIR` is process-global and the suite is threaded, so setting it in-process
/// would reach every other test running at that moment. Re-executing this binary
/// with `TMPDIR` pointed at a booby-trapped directory is the isolation. If a trap
/// ever stops biting — because the cache path is spelled differently — the child's
/// `expect_err` fires and this goes red; it cannot go quietly green.
///
/// No child reaches `uv`: the `?` short-circuits before the spawn, which is why
/// this runs on a CI machine that has none.
#[test]
fn a_script_cache_that_cannot_be_written_stops_the_step() {
// The child half: TMPDIR is already trapped, so materialising must fail. Which
// operator to drive arrives in the same variable that marks this as the child.
if let Ok(which) = std::env::var("ARC_SCRIPT_CACHE_TRAP") {
let protocol = tempfile::tempdir().unwrap();
let env = HashMap::new();
let ctx = test_ctx(protocol.path(), &env);
let (err, cache_dir) = match which.as_str() {
"umap_project" => {
let with: Value = serde_yaml::from_str(
"input: homes.parquet\ncolumns: [longitude]\nout: out.parquet",
)
.unwrap();
(
UmapProject
.run(&with, &ctx)
.expect_err("a script cache that cannot be written must stop the step"),
UMAP_PROJECT_CACHE_DIR,
)
}
"text_embed" => {
// The extension check runs BEFORE the script is materialised, so it
// has to pass or the refusal under test is never reached.
std::fs::write(protocol.path().join("subtoken.ext"), b"not real").unwrap();
let with: Value = serde_yaml::from_str(
"input: corpus.parquet\ntext_column: description\nextension: subtoken.ext\nout: out.parquet",
)
.unwrap();
(
TextEmbed
.run(&with, &ctx)
.expect_err("a script cache that cannot be written must stop the step"),
TEXT_EMBED_CACHE_DIR,
)
}
"splink_resolve" => {
let with: Value = serde_yaml::from_str(
"edgar: edgar.parquet\ngleif: gleif.parquet\nout: out.parquet",
)
.unwrap();
(
SplinkResolve
.run(&with, &ctx)
.expect_err("a script cache that cannot be written must stop the step"),
SPLINK_RESOLVE_CACHE_DIR,
)
}
"gleif_ra_fetch" => {
let with: Value = serde_yaml::from_str("ra: RA000665\nout: out.csv").unwrap();
(
GleifRaFetch
.run(&with, &ctx)
.expect_err("a script cache that cannot be written must stop the step"),
GLEIF_RA_FETCH_CACHE_DIR,
)
}
other => panic!("no operator called `{other}` to drive"),
};
let msg = err.to_string();
assert!(
msg.contains(cache_dir),
"the refusal names the cache it could not write: {msg}"
);
return;
}
// The parent half: trap the cache two ways per operator and re-run this one
// test under each.
//
// `arcform-op-<name>-1.0.0` as a regular FILE blocks `create_dir_all`; as a
// directory holding a DIRECTORY called `<name>.py`, it lets the directory be
// created and blocks the write instead.
//
// `--exact` matches the test's FULL path, so the module prefix is not optional.
// Getting it wrong does not fail — libtest runs zero tests and exits 0 — which
// is why the assertion below reads what the child REPORTED and not only how it
// exited. A child that ran nothing is not a child that passed.
const CHILD: &str = "operator::tests::a_script_cache_that_cannot_be_written_stops_the_step";
for (operator, cache_dir, script) in [
("umap_project", UMAP_PROJECT_CACHE_DIR, "umap_project.py"),
("text_embed", TEXT_EMBED_CACHE_DIR, "text_embed.py"),
(
"splink_resolve",
SPLINK_RESOLVE_CACHE_DIR,
"splink_resolve.py",
),
(
"gleif_ra_fetch",
GLEIF_RA_FETCH_CACHE_DIR,
"gleif_ra_fetch.py",
),
] {
let block_the_directory = tempfile::tempdir().unwrap();
std::fs::write(block_the_directory.path().join(cache_dir), b"in the way").unwrap();
let block_the_write = tempfile::tempdir().unwrap();
std::fs::create_dir_all(block_the_write.path().join(cache_dir).join(script)).unwrap();
for trap in [block_the_directory.path(), block_the_write.path()] {
let out = std::process::Command::new(std::env::current_exe().unwrap())
.args([CHILD, "--exact", "--nocapture"])
.env("TMPDIR", trap)
.env("ARC_SCRIPT_CACHE_TRAP", operator)
.output()
.expect("re-run this test binary");
let said = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
assert!(
said.contains("1 passed"),
"the child has to have RUN this test, not merely exited 0 — a filter \
that matches nothing reports `0 passed` and exits 0. {operator} at \
trap {}:\n{said}",
trap.display(),
);
assert!(
out.status.success(),
"with {operator}'s script cache trapped at {}, materialising must \
fail and the step must stop:\n{said}",
trap.display(),
);
}
}
}
/// The reproducibility contract, read off the frozen bytes `op@1` addresses: the
/// seed is in the script, and the script consults no credential and imports no
/// HTTP client — so a run's cost is the machine it runs on.
#[test]
fn umap_project_script_pins_its_seed_and_reads_no_credentials() {
assert!(
UMAP_PROJECT_PY.contains("SEED = 42"),
"the seed is frozen into the script, not exposed in `with:`"
);
assert!(
UMAP_PROJECT_PY.contains("random_state=SEED"),
"the seed has to reach the projection, or two runs diverge"
);
for forbidden in [
"os.getenv",
"environ.get",
"import requests",
"import urllib",
"import httpx",
"import socket",
] {
assert!(
!UMAP_PROJECT_PY.contains(forbidden),
"umap_project.py must not carry `{forbidden}`: the step reads no key \
and opens no socket"
);
}
}
/// [`UMAP_METRICS`] is the one place the authoring schema's `enum` (`with_schema`)
/// and this operator's `validate` both read from — those two cannot disagree with
/// each other. What they COULD still disagree with is the frozen script's own
/// tuple, which is a separate literal in a different language: narrowing one side
/// without the other lets a manifest validate a metric the script's argparse then
/// refuses (or the reverse — a manifest refused for a metric the script would
/// have accepted). Parsed out of the embedded bytes `op@1` addresses rather than
/// copied by hand, so that divergence reddens here instead of an hour into a run.
#[test]
fn umap_project_metrics_list_agrees_with_the_script() {
// ANCHORED AT THE START OF A LINE, not merely found anywhere. A bare
// `.find("METRICS = (")` takes the first SUBSTRING, so a comment line above
// the real tuple reads as the tuple — `# UMAP_METRICS = ("euclidean",)` above
// it defeated this assertion entirely — and so does the string
// `UMAP_METRICS = (` itself, which contains the needle. A pin at the shape of
// the source is not a pin at what the source does, and a source-text scan
// cannot see through indirection; anchoring is the cheapest thing that makes
// this one read the definition rather than a mention of it.
let needle = "\nMETRICS = (";
let start = UMAP_PROJECT_PY
.find(needle)
.expect("umap_project.py must define METRICS = (...) at the start of a line")
+ needle.len();
assert!(
UMAP_PROJECT_PY[start..].find(needle).is_none(),
"umap_project.py defines METRICS more than once at line start; this \
assertion would silently pin whichever came first"
);
let end = UMAP_PROJECT_PY[start..]
.find(')')
.expect("the METRICS tuple must close")
+ start;
let script_metrics: Vec<&str> = UMAP_PROJECT_PY[start..end]
.split(',')
.map(|s| s.trim().trim_matches('"'))
.filter(|s| !s.is_empty())
.collect();
assert_eq!(
script_metrics,
UMAP_METRICS.to_vec(),
"umap_project.py's METRICS tuple has diverged from UMAP_METRICS in \
src/operator.rs, which the authoring schema's `enum` and this \
operator's `validate` both read from"
);
}
/// The authoring schema's `metric` enum must BE [`UMAP_METRICS`], not merely
/// resemble it.
///
/// [`umap_project_metrics_list_agrees_with_the_script`] pins the frozen script's
/// tuple against that constant. Nothing pinned the third reader. Replacing
/// `"enum": UMAP_METRICS` with a hand-written literal, or deleting the key
/// outright, left the whole workspace green — and this schema is what `arc mcp`'s
/// `operator_describe` emits, so an authoring client would offer a metric set
/// that the operator's own validation then refuses.
#[test]
fn umap_project_schema_enum_is_the_metrics_constant() {
let schema = with_schema("umap_project").expect("umap_project has a with: schema");
let declared = schema["properties"]["metric"]["enum"]
.as_array()
.expect("the metric property must declare an enum, or a manifest can name anything");
let declared: Vec<&str> = declared.iter().map(|v| v.as_str().unwrap()).collect();
assert_eq!(
declared,
UMAP_METRICS.to_vec(),
"the authoring schema's metric enum has diverged from UMAP_METRICS, which \
the operator's own validation reads from"
);
}
/// The `neighbors` schema states the bound the input's row count sets, twice: as
/// `x-at-most`, which a program reads to work the bound out on a table without a
/// copy of the rule, and in the description, which editor completion and
/// `operator_describe` show a person. Both say the bound applies to the default as
/// well, so an unset value on a table of 10 rows reads as 9, not 15.
#[test]
fn umap_project_schema_states_the_neighbors_bound_and_the_default_rule() {
let schema = with_schema("umap_project").expect("umap_project has a with: schema");
let neighbors = &schema["properties"]["neighbors"];
assert_eq!(
neighbors["minimum"], 2,
"the lower bound is UMAP's: {neighbors}"
);
assert_eq!(
neighbors["default"], 15,
"the default is the script's DEFAULT_NEIGHBORS: {neighbors}"
);
assert_eq!(
neighbors["x-at-most"],
serde_json::json!({ "rows_of": "input", "minus": 1, "bounds_default": true }),
"the bound is a rule on the row count of the Parquet `input` names, and it \
bounds the default: {neighbors}"
);
let described = neighbors["description"]
.as_str()
.expect("neighbors carries a description");
for said in [
"one below the input's row count",
"refuses a value above that bound",
"An unset value takes 15 or that bound, whichever is smaller",
] {
assert!(
described.contains(said),
"the neighbors description must say {said:?} — an authoring client \
shows this string and never the README: {described}"
);
}
}
/// A Parquet of `rows` rows at `dir/name`, written by DuckDB in process.
fn parquet_of_rows(dir: &Path, name: &str, rows: u64) -> PathBuf {
let path = dir.join(name);
let conn = duckdb::Connection::open_in_memory().unwrap();
conn.execute_batch(&format!(
"COPY (SELECT range AS id, range * 0.5 AS x FROM range({rows})) TO '{}' \
(FORMAT parquet);",
sql_lit(&path.display().to_string())
))
.unwrap();
path
}
fn umap_config_with_neighbors(neighbors: Option<u64>) -> UmapProjectConfig {
UmapProjectConfig {
input: "in.parquet".to_string(),
columns: vec!["x".to_string()],
out: "maps/out.parquet".to_string(),
fit: Some("maps/fit.umap".to_string()),
neighbors,
min_dist: None,
metric: None,
}
}
#[test]
fn parquet_row_count_counts_every_row_and_reads_none_from_a_missing_file() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(
parquet_row_count(&parquet_of_rows(dir.path(), "in.parquet", 48)),
Some(48)
);
assert_eq!(
parquet_row_count(&parquet_of_rows(dir.path(), "empty.parquet", 0)),
Some(0)
);
assert_eq!(parquet_row_count(&dir.path().join("absent.parquet")), None);
}
/// The bound is one below the row count, exactly: on 48 rows 47 runs and 48 is
/// refused, naming the value, the bound, the row count and the input.
#[test]
fn umap_project_refuses_neighbors_above_one_below_the_input_row_count() {
let dir = tempfile::tempdir().unwrap();
parquet_of_rows(dir.path(), "in.parquet", 48);
umap_config_with_neighbors(Some(47))
.refuse_neighbors_past_the_input(dir.path())
.expect("47 is one below 48 rows, the bound itself, and runs");
for asked in [48, 60] {
let err = umap_config_with_neighbors(Some(asked))
.refuse_neighbors_past_the_input(dir.path())
.expect_err("a neighbors above one below the row count must be refused");
let Error::ManifestValidation(said) = &err else {
panic!("the refusal must be ManifestValidation, which is not retried: {err:?}");
};
assert!(
said.contains(&format!(
"`neighbors: {asked}` is above 47, one below the 48 rows of in.parquet"
)),
"the refusal names the value, the bound, the row count and the input: {said}"
);
}
assert!(
!dir.path().join("maps").exists(),
"the refusal writes nothing"
);
}
/// An unset `neighbors` is not counted against the input: on 10 rows it runs, and
/// the script's clamp gives it 9. And an input that cannot be read is left to the
/// script, which fails naming it.
#[test]
fn umap_project_leaves_an_unset_neighbors_and_an_unreadable_input_to_the_script() {
let dir = tempfile::tempdir().unwrap();
parquet_of_rows(dir.path(), "in.parquet", 10);
umap_config_with_neighbors(None)
.refuse_neighbors_past_the_input(dir.path())
.expect("an unset neighbors is never refused for the row count");
let absent = tempfile::tempdir().unwrap();
umap_config_with_neighbors(Some(200))
.refuse_neighbors_past_the_input(absent.path())
.expect("an input arc cannot read is the script's to refuse, not this check's");
}
// ── text_embed ───────────────────────────────────────────────────────────
/// The EXTENSION is an INPUT, recorded beside the corpus rather than installed —
/// this is what puts the Protocol's dependency on one particular build of the
/// embedder into the graph. It decides what a vector is, so swapping it has to
/// re-run the embedding exactly as swapping the corpus does.
#[test]
fn text_embed_declares_the_extension_as_a_read_beside_the_corpus() {
let with: Value = serde_yaml::from_str(
"input: build/corpus.parquet\ntext_column: description\nextension: vendor/subtoken.duckdb_extension\nout: build/embedded.parquet",
)
.unwrap();
let assets = assets_for("text_embed", Some(&with)).unwrap();
assert_eq!(
assets.reads,
vec!["build/corpus.parquet", "vendor/subtoken.duckdb_extension"]
);
assert_eq!(assets.produces, vec!["build/embedded.parquet"]);
assert_eq!(
assets.kinds.get("vendor/subtoken.duckdb_extension"),
Some(&crate::asset_kind::AssetKind::File),
"the extension is one file, hashed for staleness like the corpus"
);
assert_eq!(
assets.kinds.get("build/corpus.parquet"),
Some(&crate::asset_kind::AssetKind::File)
);
assert_eq!(
assets.kinds.get("build/embedded.parquet"),
Some(&crate::asset_kind::AssetKind::File)
);
}
/// The model directory is OPTIONAL and, when declared, is a read like any other —
/// so editing a file inside it re-runs the step rather than leaving a check that
/// passed against bytes that are no longer there.
#[test]
fn text_embed_records_a_declared_model_as_a_directory_read_and_omits_it_otherwise() {
let base = "input: c.parquet\ntext_column: description\nextension: e.ext\nout: o.parquet\n";
let without: Value = serde_yaml::from_str(base).unwrap();
assert_eq!(
assets_for("text_embed", Some(&without)).unwrap().reads,
vec!["c.parquet", "e.ext"],
"no model declared means no model asset — the weights are in the extension"
);
let with: Value = serde_yaml::from_str(&format!(
"{base}model: models/potion\nmodel_release: minishlab/potion-base-8M@abcdef"
))
.unwrap();
let assets = assets_for("text_embed", Some(&with)).unwrap();
assert_eq!(assets.reads, vec!["c.parquet", "e.ext", "models/potion"]);
assert_eq!(
assets.kinds.get("models/potion"),
Some(&crate::asset_kind::AssetKind::Directory),
"a declared model is a directory, so staleness hashes what is in it"
);
}
/// `model:` and `model_release:` are two halves of one check — the extension
/// hashes the release identity together with the files, so a directory without a
/// release names bytes whose address cannot be computed, and a release without a
/// directory names an address with nothing to compute it from. Either alone is a
/// check that silently does not happen, which is why it is refused at load.
#[test]
fn text_embed_refuses_half_a_model_declaration() {
let base = "input: c.parquet\ntext_column: description\nextension: e.ext\nout: o.parquet\n";
for half in [
"model: models/potion",
"model_release: minishlab/potion-base-8M@abcdef",
] {
let with: Value = serde_yaml::from_str(&format!("{base}{half}")).unwrap();
let err = assets_for("text_embed", Some(&with)).expect_err(&format!(
"`{half}` alone checks nothing and must be refused"
));
assert!(
err.to_string().contains("Set both, or neither"),
"the refusal says what to do about it: {err}"
);
}
let both: Value = serde_yaml::from_str(&format!(
"{base}model: models/potion\nmodel_release: minishlab/potion-base-8M@abcdef"
))
.unwrap();
assert!(
assets_for("text_embed", Some(&both)).is_ok(),
"both halves together are the point of the fields and must load"
);
}
#[test]
fn text_embed_rejects_bad_config() {
for missing in [
"text_column: t\nextension: e\nout: o",
"input: i\nextension: e\nout: o",
"input: i\ntext_column: t\nout: o",
"input: i\ntext_column: t\nextension: e",
] {
let with: Value = serde_yaml::from_str(missing).unwrap();
assert!(
assets_for("text_embed", Some(&with)).is_err(),
"a `with:` block missing a required field must not load: {missing}"
);
}
let bogus: Value =
serde_yaml::from_str("input: i\ntext_column: t\nextension: e\nout: o\nbogus: 1")
.unwrap();
assert!(assets_for("text_embed", Some(&bogus)).is_err());
}
/// `vector_column:` names the column the vectors land in, so a blank one names
/// nothing. Refused at load rather than by DuckDB's parser inside the script.
#[test]
fn text_embed_refuses_a_vector_column_that_is_not_a_name() {
let base = "input: i\ntext_column: t\nextension: e\nout: o\n";
for blank in ["vector_column: \"\"", "vector_column: \" \""] {
let with: Value = serde_yaml::from_str(&format!("{base}{blank}")).unwrap();
let err = assets_for("text_embed", Some(&with))
.expect_err(&format!("`{blank}` names no column and must be refused"));
assert!(
err.to_string().contains("is blank"),
"the refusal says what is wrong with it: {err}"
);
}
let named: Value =
serde_yaml::from_str(&format!("{base}vector_column: title_vector")).unwrap();
assert!(
assets_for("text_embed", Some(&named)).is_ok(),
"a named vector column is the point of the field and must load"
);
}
/// An unset `vector_column` is OMITTED from the argv entirely, so the frozen
/// script's own default stays the single place it is written down.
#[test]
fn text_embed_args_omit_the_vector_column_the_manifest_did_not_set() {
assert_eq!(
text_embed_args("c.parquet", "description", "e.ext", "o.parquet", None, None),
vec![
"--input",
"c.parquet",
"--text-column",
"description",
"--extension",
"e.ext",
"--out",
"o.parquet"
],
);
}
#[test]
fn text_embed_args_pass_the_vector_column_through() {
assert_eq!(
text_embed_args(
"c.parquet",
"description",
"e.ext",
"o.parquet",
Some("title_vector"),
None
),
vec![
"--input",
"c.parquet",
"--text-column",
"description",
"--extension",
"e.ext",
"--out",
"o.parquet",
"--vector-column",
"title_vector"
],
);
}
/// An undeclared model is OMITTED from the argv entirely rather than passed as an
/// empty string — the script's model check is skipped by its absence, and a blank
/// path would be a directory it went looking for and did not find.
#[test]
fn text_embed_args_carry_the_model_declaration_only_when_there_is_one() {
let release = "minishlab/potion-base-8M@bf8b056651a2c21b8d2565580b8569da283cab23";
assert_eq!(
text_embed_args(
"c.parquet",
"description",
"e.ext",
"o.parquet",
None,
Some(("models/potion", release))
),
vec![
"--input",
"c.parquet",
"--text-column",
"description",
"--extension",
"e.ext",
"--out",
"o.parquet",
"--model",
"models/potion",
"--model-release",
release
],
);
assert!(
!text_embed_args("c.parquet", "description", "e.ext", "o.parquet", None, None)
.iter()
.any(|a| a == "--model" || a == "--model-release"),
"no declaration means neither flag reaches the script"
);
}
/// The check answers with the path that is absent, because that path is the
/// message: the fix is to run the step that puts it there.
#[test]
fn missing_model_part_names_the_file_that_is_absent() {
let tmp = tempfile::tempdir().unwrap();
let model = tmp.path().join("model");
assert_eq!(
missing_model_part(&model),
Some(model.clone()),
"a model directory that was never fetched is named by its own path"
);
std::fs::create_dir_all(&model).unwrap();
std::fs::write(model.join("tokenizer.json"), "{}").unwrap();
assert_eq!(
missing_model_part(&model),
Some(model.join("model.safetensors")),
"a half-extracted model names the file that did not arrive"
);
std::fs::write(model.join("model.safetensors"), b"\0").unwrap();
assert_eq!(
missing_model_part(&model),
Some(model.join("config.json")),
"the weights and the tokenizer are not enough: the extension's model \
address is computed over three files, and two of them cannot reproduce it"
);
std::fs::write(model.join("config.json"), "{}").unwrap();
assert_eq!(
missing_model_part(&model),
None,
"all three parts present — nothing is missing"
);
std::fs::remove_file(model.join("tokenizer.json")).unwrap();
assert_eq!(
missing_model_part(&model),
Some(model.join("tokenizer.json")),
"the tokenizer is checked too, not just the weights"
);
// A `model:` that is there but is NOT a directory passes this check and is
// left to the script to read. The directory layout is the operator's
// contract; a caller pointing at something else gets the script's reading of
// it rather than a guess made here about what it should have contained.
let single_file = tmp.path().join("model.bin");
std::fs::write(&single_file, b"\0").unwrap();
assert_eq!(
missing_model_part(&single_file),
None,
"a model path that exists and is not a directory is not this check's to \
refuse"
);
}
/// Without a `uv` on PATH: the step refuses on the missing extension and never
/// gets as far as anything that could be mistaken for a fetch or an install.
#[test]
fn text_embed_refuses_a_missing_extension_before_it_spawns_uv() {
let tmp = tempfile::tempdir().unwrap();
let env = HashMap::new();
let ctx = test_ctx(tmp.path(), &env);
let with: Value = serde_yaml::from_str(
"input: corpus.parquet\ntext_column: description\nextension: vendor/subtoken.duckdb_extension\nout: build/embedded.parquet",
)
.unwrap();
let err = TextEmbed
.run(&with, &ctx)
.expect_err("an extension that is not on disk must stop the step");
assert!(
matches!(err, Error::StepExecution { .. }),
"a missing extension is deterministic, so it must be the NON-retryable \
kind rather than a StepFailed the engine would back off and re-attempt: \
{err:?}"
);
let msg = err.to_string();
assert!(
msg.contains("vendor/subtoken.duckdb_extension"),
"the refusal names the declared extension: {msg}"
);
assert!(
msg.contains("does not download one"),
"the refusal says the artifact is the Protocol's job, not the operator's: \
{msg}"
);
assert!(
msg.contains("does not install one from a registry"),
"an extension has a second way to arrive by itself, and the refusal has \
to close that one too: {msg}"
);
}
/// A DECLARED model that cannot reproduce the extension's address stops the step
/// before `uv`, naming the file that makes it impossible. The alternative — going
/// ahead and reporting that the address did not match — is true and useless: the
/// reader cannot tell a wrong model from an incomplete directory.
#[test]
fn text_embed_refuses_a_model_directory_that_cannot_reproduce_the_address() {
let tmp = tempfile::tempdir().unwrap();
let env = HashMap::new();
let ctx = test_ctx(tmp.path(), &env);
std::fs::write(tmp.path().join("subtoken.ext"), b"not real").unwrap();
let model = tmp.path().join("models/potion");
std::fs::create_dir_all(&model).unwrap();
// The two files the operator used to want, and one short of what the address
// is computed over.
std::fs::write(model.join("tokenizer.json"), "{}").unwrap();
std::fs::write(model.join("model.safetensors"), b"\0").unwrap();
let with: Value = serde_yaml::from_str(
"input: corpus.parquet\ntext_column: description\nextension: subtoken.ext\nout: build/embedded.parquet\nmodel: models/potion\nmodel_release: minishlab/potion-base-8M@abcdef",
)
.unwrap();
let err = TextEmbed
.run(&with, &ctx)
.expect_err("a model directory one file short must stop the step");
let msg = err.to_string();
assert!(
msg.contains("config.json"),
"the refusal names the file that is missing, not merely that a check \
failed: {msg}"
);
assert!(
msg.contains("models/potion"),
"the refusal names the declared model: {msg}"
);
}
/// What is handed to `uv`, decided without spawning it — the same check the
/// projection gets, for the same reason.
///
/// `vector_column:` is set here, to a name distinct from every other string in
/// this manifest, for the same reason the projection's three knobs are set to
/// distinct values in its own version of this test: `text_embed_args_*` already
/// proves the pure builder emits `--vector-column` when handed one directly, but
/// nothing before this test drove `cfg.vector_column.as_deref()` at the call site
/// in `text_embed_invocation` — a manifest that never sets it cannot tell
/// "threaded" from "silently dropped" in the resulting argv.
#[test]
fn text_embed_invocation_materialises_the_frozen_script_and_names_it() {
let tmp = tempfile::tempdir().unwrap();
let model = tmp.path().join("models/potion");
std::fs::create_dir_all(&model).unwrap();
for part in MODEL_PARTS {
std::fs::write(model.join(part), b"not a real model").unwrap();
}
let extension = tmp.path().join("vendor/subtoken.duckdb_extension");
std::fs::create_dir_all(extension.parent().unwrap()).unwrap();
std::fs::write(&extension, b"not a real extension").unwrap();
let release = "minishlab/potion-base-8M@bf8b056651a2c21b8d2565580b8569da283cab23";
let with: Value = serde_yaml::from_str(&format!(
"input: build/corpus.parquet\ntext_column: description\nextension: vendor/subtoken.duckdb_extension\nout: build/embedded.parquet\nvector_column: description_vector\nmodel: models/potion\nmodel_release: {release}"
))
.unwrap();
let cfg = TextEmbedConfig::parse(&with).unwrap();
let args =
text_embed_invocation(&cfg, tmp.path()).expect("a complete declaration proceeds");
assert_eq!(args[0], "run");
assert_eq!(args[1], "--script");
let script = std::path::Path::new(&args[2]);
assert_eq!(
std::fs::read_to_string(script).unwrap(),
TEXT_EMBED_PY,
"`op@1` addresses the embedded bytes: what lands on disk is what is \
compiled in, not a script found by path"
);
assert_eq!(
&args[3..],
[
"--input",
&tmp.path()
.join("build/corpus.parquet")
.display()
.to_string(),
"--text-column",
"description",
"--extension",
&extension.display().to_string(),
"--out",
&tmp.path()
.join("build/embedded.parquet")
.display()
.to_string(),
"--vector-column",
"description_vector",
"--model",
&model.display().to_string(),
"--model-release",
release,
],
"every path reaches the script resolved against the protocol directory, \
and `vector_column:` reaches the invocation with the name the manifest \
set — not just the name `text_embed_args` would emit if handed it \
directly"
);
}
/// The step reads no credential and opens no socket — the same scan the
/// projection gets, over the bytes `op@1` addresses.
#[test]
fn text_embed_script_reads_no_credentials() {
for forbidden in [
"os.getenv",
"environ.get",
"environ[\"OPENAI",
"import requests",
"import urllib",
"import httpx",
"import socket",
] {
assert!(
!TEXT_EMBED_PY.contains(forbidden),
"text_embed.py must not carry `{forbidden}`: the step reads no key \
and opens no socket"
);
}
}
/// This operator is a stopgap and its own source has to say so, naming where the
/// capability is going — otherwise the next person to read it hardens a temporary
/// arrangement, which is how a second implementation of one capability becomes
/// permanent.
///
/// A string assertion, deliberately: the thing being pinned IS text, and deleting
/// the notice while keeping the operator is exactly the change this has to catch.
///
/// SCOPED TO THE OPENING OF THE DOCSTRING, not to the file, and that is the whole
/// difference between this test and a test that cannot fail. An assertion over the
/// whole file was GREEN with the notice deleted — measured 2026-08-24 by deleting
/// it — because `DuckDB extension` also occurs further down, in the paragraph
/// about the implementation order that the notice happens to sit above. What is
/// asserted here is that a reader opening this file meets the warning before
/// anything else, which is the only form of it that does any work.
#[test]
fn text_embed_marks_itself_provisional_and_names_where_it_is_going() {
let docstring = TEXT_EMBED_PY
.split("\"\"\"")
.nth(1)
.expect("text_embed.py opens with a module docstring");
let notice = docstring.lines().take(8).collect::<Vec<_>>().join("\n");
assert!(
notice.contains("PROVISIONAL"),
"text_embed.py must open by announcing that it is a stopgap:\n{notice}"
);
assert!(
notice.contains("DuckDB embedding extension"),
"the notice must name the DuckDB embedding extension as where this \
capability is going, not merely say it is temporary:\n{notice}"
);
assert!(
notice.contains("DELETED"),
"the notice must say the operator goes away rather than being ported:\n\
{notice}"
);
}
// ── finetype_validate ───────────────────────────────────────────────────
#[test]
fn finetype_validate_is_in_catalog_and_versioned() {
assert!(resolve("finetype_validate").is_ok());
assert!(resolve("finetype_validate@1").is_ok());
assert!(resolve("finetype_validate@^1.0").is_ok());
assert!(resolve("finetype_validate@2").is_err()); // version not satisfied
}
#[test]
fn finetype_validate_declares_reads_and_no_produces() {
// Gate: reads [parquet, schema] (parquet first → downstream of its producer),
// produces nothing (check-only leaves the terminal Parquet byte-identical).
let with: Value = serde_yaml::from_str(
"parquet: build/edgar_gleif.parquet\nschema: schema.finetype.json",
)
.unwrap();
let a = assets_for("finetype_validate", Some(&with)).unwrap();
assert_eq!(
a.reads,
vec![
"build/edgar_gleif.parquet".to_string(),
"schema.finetype.json".to_string(),
]
);
assert!(a.produces.is_empty());
}
#[test]
fn finetype_validate_rejects_bad_config() {
// missing required `schema`
let no_schema: Value = serde_yaml::from_str("parquet: p.parquet").unwrap();
assert!(assets_for("finetype_validate", Some(&no_schema)).is_err());
// missing required `parquet`
let no_parquet: Value = serde_yaml::from_str("schema: s.json").unwrap();
assert!(assets_for("finetype_validate", Some(&no_parquet)).is_err());
// unknown field rejected (deny_unknown_fields)
let bogus: Value = serde_yaml::from_str("parquet: p\nschema: s\nbogus: 1").unwrap();
assert!(assets_for("finetype_validate", Some(&bogus)).is_err());
}
/// The finetype DuckDB extension the in-engine gate LOADs, or `None` to skip.
///
/// `ci.yml` stages the published community build and sets
/// `ARCFORM_REQUIRE_FINETYPE_EXT=1`. With that set a missing extension PANICS
/// instead of skipping: a staging step that is deleted, renamed or silently starts
/// resolving nothing would otherwise turn every test below back into a skip, and a
/// skip reads as `ok` to everything except a human counting the test summary.
fn finetype_ext_or_skip() -> Option<String> {
match std::env::var("FINETYPE_DUCKDB_EXT") {
Ok(p) if std::path::Path::new(&p).exists() => Some(p),
found => {
assert!(
std::env::var_os("ARCFORM_REQUIRE_FINETYPE_EXT").is_none(),
"ARCFORM_REQUIRE_FINETYPE_EXT is set, so this run promised a staged \
finetype extension, but FINETYPE_DUCKDB_EXT is {found:?} — the \
staging step is gone or points at nothing"
);
eprintln!("skipping finetype_validate: FINETYPE_DUCKDB_EXT unset or missing");
None
}
}
}
/// Write a Parquet from a `SELECT` and return its path — the gate reads Parquet via
/// `read_parquet`, so the fixtures are materialised by DuckDB itself.
fn write_parquet(dir: &Path, name: &str, select: &str) -> PathBuf {
let path = dir.join(name);
let conn = duckdb::Connection::open_in_memory().unwrap();
conn.execute_batch(&format!(
"COPY ({select}) TO '{}' (FORMAT parquet);",
sql_lit(&path.display().to_string())
))
.unwrap();
path
}
/// Run `finetype_validate` over `parquet` against `schema`, returning the stderr the
/// runner surfaces (`StepFailed.stderr`, which is what `Error`'s Display prints) or
/// the `StepOutput` of a clean run.
fn run_validate(
dir: &Path,
ext: &str,
parquet: &str,
schema: &str,
) -> std::result::Result<StepOutput, String> {
let env = HashMap::new();
let ctx = test_ctx(dir, &env);
let op = resolve("finetype_validate").unwrap();
let with: Value = serde_yaml::from_str(&format!(
"parquet: {parquet}\nschema: {schema}\nextension: {ext}"
))
.unwrap();
match op.run(&with, &ctx) {
Ok(out) => Ok(out),
Err(Error::StepFailed { stderr, .. }) => Err(stderr),
other => panic!("expected Ok or StepFailed, got {:?}", other.map(|_| ())),
}
}
/// A minimal Parquet + a JSON Schema it satisfies, then a schema it violates — the
/// two halves of the in-engine gate. Needs the built finetype DuckDB extension: the
/// verify harness exports `FINETYPE_DUCKDB_EXT` to point at it; if it is unset or
/// absent (e.g. CI without the extension) the test SKIPS rather than false-fails.
#[test]
fn finetype_validate_passes_clean_and_fails_on_drift() {
let Some(ext) = finetype_ext_or_skip() else {
return;
};
let dir = tempfile::tempdir().unwrap();
// Build a tiny Parquet with one column `code` holding two 2-letter values — the
// gate reads Parquet via `read_parquet`, so materialise it with DuckDB itself.
let parquet = dir.path().join("data.parquet");
{
let conn = duckdb::Connection::open_in_memory().unwrap();
conn.execute_batch(&format!(
"COPY (SELECT 'US' AS code UNION ALL SELECT 'GB') TO '{}' (FORMAT parquet);",
sql_lit(&parquet.display().to_string())
))
.unwrap();
}
// PASS schema: a closed enum containing BOTH values — 0 rejects.
std::fs::write(
dir.path().join("pass.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"code":{"type":"string","minLength":2,"maxLength":2,"enum":["US","GB"]}}}"#,
)
.unwrap();
// DRIFT schema: a closed enum that does NOT contain "GB" — the second row
// violates it, so ft_validate reports a reject and the gate must fail closed.
std::fs::write(
dir.path().join("drift.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"code":{"type":"string","enum":["US"]}}}"#,
)
.unwrap();
// Wire the extension through the per-step `extension:` config field — the same
// resolution path the env var feeds, without mutating process-global env.
let ext_yaml = format!("\nextension: {}", ext);
let env = HashMap::new();
let ctx = test_ctx(dir.path(), &env);
let op = resolve("finetype_validate").unwrap();
let pass_with: Value = serde_yaml::from_str(&format!(
"parquet: data.parquet\nschema: pass.json{ext_yaml}"
))
.unwrap();
match op.run(&pass_with, &ctx) {
Ok(_) => {} // clean → gate passes
other => panic!("clean validate should pass, got {:?}", other.map(|_| ())),
}
let drift_with: Value = serde_yaml::from_str(&format!(
"parquet: data.parquet\nschema: drift.json{ext_yaml}"
))
.unwrap();
match op.run(&drift_with, &ctx) {
// Fail-closed: a row violates the contract → rejects > 0 → StepFailed.
Err(Error::StepFailed { .. }) => {}
other => panic!("drift must fail the gate, got {:?}", other.map(|_| ())),
}
}
/// A real refusal, end to end, asserted on the stderr the runner surfaces — not on
/// what a formatting function returns.
///
/// The fixture has TWO columns and only one of them drifts, so the assertions pin
/// WHICH column is named and WHICH value is quoted, rather than that some name and
/// some value are present. A sample drawn from the wrong column reddens this on both
/// halves: the drifted value goes missing and the clean column's value appears.
///
/// It runs the accepting schema FIRST and requires it to pass. Without that pairing
/// the whole test is satisfied by a validator that refuses every run for the wrong
/// reason.
#[test]
fn finetype_validate_drift_names_the_column_the_clause_and_the_value() {
let Some(ext) = finetype_ext_or_skip() else {
return;
};
let dir = tempfile::tempdir().unwrap();
write_parquet(
dir.path(),
"data.parquet",
"SELECT * FROM (VALUES \
('ISSUED', 'Acme Holdings'), \
('ISSUED', 'Beta Trading'), \
('PENDING_ARCHIVAL', 'Globex Systems')) t(reg_status, legal_name)",
);
// Accepting contract: the enum lists both statuses, the name is unconstrained
// beyond a length floor. Zero rejects.
std::fs::write(
dir.path().join("pass.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{
"reg_status":{"type":"string","enum":["ISSUED","PENDING_ARCHIVAL"]},
"legal_name":{"type":"string","minLength":1}}}"#,
)
.unwrap();
// The same contract with ONE value dropped from the enum — the shape of the
// real drift: a live upstream status the committed enum never listed.
std::fs::write(
dir.path().join("drift.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{
"reg_status":{"type":"string","enum":["ISSUED"]},
"legal_name":{"type":"string","minLength":1}}}"#,
)
.unwrap();
// The pairing: the accepting run must pass, and pass SILENTLY — no new
// output on the happy path, same empty StepOutput as before. Without it a
// validator that refuses every run satisfies everything below.
let clean = run_validate(dir.path(), &ext, "data.parquet", "pass.json")
.expect("the accepting contract must pass");
// Two assertions, not one conjunction: `&&` folded to `||` is a real mutation
// of this line, and it leaves an assertion that passes when either half holds.
assert!(
clean.stderr.is_empty(),
"a passing validate run writes no stderr, got {clean:?}"
);
assert!(
clean.stdout.is_none(),
"a passing validate run writes no stdout, got {clean:?}"
);
let stderr = run_validate(dir.path(), &ext, "data.parquet", "drift.json")
.expect_err("the drifted contract must refuse");
assert!(
stderr.contains("reg_status"),
"the refusal must name the drifted column; got:\n{stderr}"
);
assert!(
stderr.contains("PENDING_ARCHIVAL"),
"the refusal must quote the offending value; got:\n{stderr}"
);
assert!(
stderr.contains("[enum]"),
"the refusal must name the clause the value failed; got:\n{stderr}"
);
// Value, the rows carrying it, and the clause as ONE rendered unit — three
// separate `contains` would be satisfied by three facts about three
// different columns.
assert!(
stderr.contains("'PENDING_ARCHIVAL' ×1 [enum]"),
"the value, its row count and its clause belong together; got:\n{stderr}"
);
assert!(
stderr.contains("1/3 row(s) reject"),
"the refusal must retain the rejecting-row count; got:\n{stderr}"
);
assert!(
stderr.contains("1 of 1 distinct"),
"the refusal must say how many distinct offenders it found; got:\n{stderr}"
);
// Nothing was cut here, so nothing may claim it was — the truncation notice
// has to track the truncation rather than always print.
assert!(
!stderr.contains("list cut"),
"a complete list must not be announced as cut; got:\n{stderr}"
);
// The clean column and its values are the control. A sample that draws from
// the wrong column, or a message that names every column rather than the
// offenders, trips one of these.
assert!(
!stderr.contains("legal_name"),
"the refusal must not name a column that did not drift; got:\n{stderr}"
);
assert!(
!stderr.contains("Globex"),
"the refusal must not quote a value from a column that did not drift; \
got:\n{stderr}"
);
}
/// A column where EVERY row rejects prints a bounded list and says so.
///
/// Twelve distinct values all violate `maxLength: 2`. The refusal must show the
/// five the bound allows, name the total it cut from, and leave the other seven
/// out — an unbounded sample would put the whole column into a step error.
#[test]
fn finetype_validate_drift_sample_is_bounded_and_says_it_was_cut() {
let Some(ext) = finetype_ext_or_skip() else {
return;
};
let dir = tempfile::tempdir().unwrap();
write_parquet(
dir.path(),
"wide.parquet",
"SELECT 'v' || lpad(i::VARCHAR, 2, '0') AS token FROM range(1, 13) t(i)",
);
std::fs::write(
dir.path().join("short.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema",
"properties":{"token":{"type":"string","maxLength":2}}}"#,
)
.unwrap();
let stderr = run_validate(dir.path(), &ext, "wide.parquet", "short.json")
.expect_err("every row violates maxLength, so the gate must refuse");
let shown = (1..=12)
.filter(|i| stderr.contains(&format!("'v{i:02}'")))
.count();
assert_eq!(
shown, DRIFT_SAMPLE_LIMIT,
"the sample is bounded at {DRIFT_SAMPLE_LIMIT} distinct values, {shown} \
appeared; got:\n{stderr}"
);
assert!(
stderr.contains("5 of 12 distinct"),
"the reader must be told how many of the total were shown; got:\n{stderr}"
);
assert!(
stderr.contains("list cut at the 5 shown"),
"the reader must be told the list was cut; got:\n{stderr}"
);
assert!(
stderr.contains("'v01' ×1 [max_length]"),
"each sampled value carries its own row count and clause; got:\n{stderr}"
);
assert!(
stderr.contains("[max_length]"),
"the refusal must name the clause; got:\n{stderr}"
);
assert!(
stderr.contains("12/12 row(s) reject"),
"the rejecting-row count is retained alongside the bounded sample; \
got:\n{stderr}"
);
}
/// The other bound: how WIDE one value may print. A cell can hold a document, and
/// five of those would be the same defect as printing the whole column.
///
/// Two offending values, one exactly `DRIFT_VALUE_WIDTH` characters and one well
/// over it. The first must print whole and the second must not, which is what
/// separates "elide past the width" from "always elide" and from "never elide" —
/// the three ways the comparison can be written.
#[test]
fn finetype_validate_elides_a_value_wider_than_the_display_bound() {
let Some(ext) = finetype_ext_or_skip() else {
return;
};
let dir = tempfile::tempdir().unwrap();
let exact = "a".repeat(DRIFT_VALUE_WIDTH);
// Past BOTH bounds: the sampled-value width and the width of ft_validate's own
// message, which quotes the same cell back inside a sentence.
let over = "b".repeat(DRIFT_VALUE_WIDTH + DRIFT_MESSAGE_WIDTH);
write_parquet(
dir.path(),
"wide.parquet",
&format!(
"SELECT * FROM (VALUES (repeat('a', {})), (repeat('b', {}))) t(blob)",
DRIFT_VALUE_WIDTH,
DRIFT_VALUE_WIDTH + DRIFT_MESSAGE_WIDTH
),
);
std::fs::write(
dir.path().join("short.json"),
r#"{"$schema":"https://json-schema.org/draft/2020-12/schema",
"properties":{"blob":{"type":"string","maxLength":2}}}"#,
)
.unwrap();
let stderr = run_validate(dir.path(), &ext, "wide.parquet", "short.json")
.expect_err("both values violate maxLength, so the gate must refuse");
// At the width: printed whole, closing quote straight after the last
// character. "Always elide" puts an ellipsis there instead.
assert!(
stderr.contains(&format!("'{exact}' ")),
"a value exactly at the display bound prints whole; got:\n{stderr}"
);
// Past the width: the head, marked as cut, and the tail nowhere.
assert!(
stderr.contains(&format!("'{}…'", "b".repeat(DRIFT_VALUE_WIDTH))),
"a value past the display bound is cut to the bound and marked; \
got:\n{stderr}"
);
// ft_validate's own message quotes the cell too, and is cut at its own bound.
assert!(
stderr.contains(&format!("\"{}…", "b".repeat(DRIFT_MESSAGE_WIDTH - 1))),
"the validator's own message is cut at its bound; got:\n{stderr}"
);
assert!(
!stderr.contains(&over),
"the whole over-wide value must not reach the message; got:\n{stderr}"
);
}
/// `ft_validate` counting rejects while the per-cell re-query finds none is a
/// disagreement between two validators, and the message says so rather than
/// printing an empty list that reads like "no values".
///
/// Asserted directly on `render` because the operator cannot be driven into that
/// state through DuckDB; the claim that the refusal reaches the operator's caller
/// at all is carried by the end-to-end tests above, not by this one.
#[test]
fn drift_sample_renders_an_empty_re_query_as_a_disagreement() {
let empty = DriftSample {
values: vec![],
distinct_total: 0,
limit: DRIFT_SAMPLE_LIMIT,
};
assert!(
empty.render().contains("none re-derived"),
"an empty sample names itself as a disagreement; got {}",
empty.render()
);
let found = DriftSample {
values: vec![("X".to_string(), "enum".to_string(), 2)],
distinct_total: 1,
limit: DRIFT_SAMPLE_LIMIT,
};
assert!(
!found.render().contains("none re-derived"),
"a sample that found values must not report a disagreement; got {}",
found.render()
);
assert!(
found.render().contains("'X' ×2 [enum]"),
"got {}",
found.render()
);
}
#[cfg(feature = "http-fetch")]
#[test]
fn http_fetch_declares_only_its_output() {
// Ingress produces the local artifact and reads no graph node (the
// network source isn't in the AssetGraph).
let hf: Value =
serde_yaml::from_str("url: https://x/edgar.parquet\nout: build/edgar.parquet").unwrap();
let a = assets_for("http_fetch", Some(&hf)).unwrap();
assert_eq!(a.produces, vec!["build/edgar.parquet".to_string()]);
assert!(a.reads.is_empty());
}
// ── html_link_discover ──────────────────────────────────────────────────
#[test]
fn archive_extract_is_in_catalog() {
assert!(resolve("archive_extract@^1.0").is_ok());
}
#[cfg(feature = "http-fetch")]
#[test]
fn html_link_discover_is_in_catalog() {
assert!(resolve("html_link_discover@1").is_ok());
assert!(resolve("html_link_discover@2").is_err()); // version not satisfied
}
#[cfg(feature = "http-fetch")]
#[test]
fn html_link_discover_declares_only_its_output() {
let with: Value = serde_yaml::from_str(
"url: https://www.sec.gov/x\npattern: 'ncen.*\\.zip'\nout: build/ncen_urls.txt",
)
.unwrap();
let a = assets_for("html_link_discover", Some(&with)).unwrap();
assert_eq!(a.produces, vec!["build/ncen_urls.txt".to_string()]);
assert!(a.reads.is_empty());
}
#[cfg(feature = "http-fetch")]
#[test]
fn html_link_discover_rejects_bad_config() {
// missing required `out`
let bad: Value = serde_yaml::from_str("url: https://x\npattern: 'a'").unwrap();
assert!(assets_for("html_link_discover", Some(&bad)).is_err());
// unknown field (deny_unknown_fields)
let bad: Value =
serde_yaml::from_str("url: https://x\npattern: 'a'\nout: o\nbogus: 1").unwrap();
assert!(assets_for("html_link_discover", Some(&bad)).is_err());
// invalid `pattern` regex is a load-time validation error
let bad: Value = serde_yaml::from_str("url: https://x\npattern: '('\nout: o").unwrap();
assert!(assets_for("html_link_discover", Some(&bad)).is_err());
}
#[test]
fn discover_links_extracts_absolutises_dedups_in_order() {
let page = "https://www.sec.gov/data-research/sec-markets-data/form-n-cen-data-sets";
let html = r#"
<a href="/files/dera/data/ncen_2024q4.zip">Q4</a>
<a href='/files/dera/data/ncen_2024q3.zip'>Q3</a>
<a href="/about">nope</a>
<a href="https://cdn.example.com/ncen_2024q2.zip">Q2 abs</a>
<a href="/files/dera/data/ncen_2024q4.zip">Q4 dup</a>
<a href="/files/dera/data/other_2024q1.zip">other</a>
"#;
let re = regex::Regex::new(r"ncen.*\.zip").unwrap();
let all = discover_links(html, page, &re, None);
assert_eq!(
all,
vec![
"https://www.sec.gov/files/dera/data/ncen_2024q4.zip".to_string(),
"https://www.sec.gov/files/dera/data/ncen_2024q3.zip".to_string(),
"https://cdn.example.com/ncen_2024q2.zip".to_string(),
]
);
// `/about` and `other_2024q1.zip` don't match `pattern`; the duplicate Q4 is
// de-duped, first-seen order preserved.
// cap: `limit` truncates to the head of the ordered, de-duped list.
let capped = discover_links(html, page, &re, Some(2));
assert_eq!(
capped,
vec![
"https://www.sec.gov/files/dera/data/ncen_2024q4.zip".to_string(),
"https://www.sec.gov/files/dera/data/ncen_2024q3.zip".to_string(),
]
);
}
#[test]
fn absolutise_handles_relative_forms() {
let page = "https://www.sec.gov/a/b/page.html";
assert_eq!(
absolutise(page, "/files/x.zip").as_deref(),
Some("https://www.sec.gov/files/x.zip")
);
// path-relative resolves against the page's directory base.
assert_eq!(
absolutise(page, "c.zip").as_deref(),
Some("https://www.sec.gov/a/b/c.zip")
);
// absolute is kept verbatim; protocol-relative borrows the page scheme.
assert_eq!(
absolutise(page, "https://cdn/x").as_deref(),
Some("https://cdn/x")
);
assert_eq!(
absolutise(page, "//cdn.example.com/x").as_deref(),
Some("https://cdn.example.com/x")
);
assert_eq!(absolutise(page, "#frag"), None);
assert_eq!(absolutise(page, ""), None);
}
// ── archive_extract ─────────────────────────────────────────────────────
#[test]
fn archive_extract_declares_members_and_reads() {
let with: Value = serde_yaml::from_str(
"archive: build/ncen/2024q1.zip\nmembers: [REGISTRANT.tsv, FUND_REPORTED_INFO.tsv]\ndest: build/ncen/2024q1",
)
.unwrap();
let a = assets_for("archive_extract", Some(&with)).unwrap();
assert_eq!(a.reads, vec!["build/ncen/2024q1.zip".to_string()]);
// Case-preserved, exactly as `members` declares it — `asset.rs` is the one
// place that lowercases for graph identity; this layer must not do it first,
// or the real on-disk spelling is gone before anything downstream sees it.
assert_eq!(
a.produces,
vec![
"build/ncen/2024q1/REGISTRANT.tsv".to_string(),
"build/ncen/2024q1/FUND_REPORTED_INFO.tsv".to_string(),
]
);
// Explicit members are individual files, not the coarse directory node.
assert_eq!(
a.kinds.get("build/ncen/2024q1/REGISTRANT.tsv"),
Some(&crate::asset_kind::AssetKind::File)
);
assert_eq!(
a.kinds.get("build/ncen/2024q1.zip"),
Some(&crate::asset_kind::AssetKind::File)
);
}
#[test]
fn archive_extract_pattern_only_declares_dest_node() {
let with: Value =
serde_yaml::from_str("archive: a.zip\npattern: '\\.tsv$'\ndest: build/out").unwrap();
let a = assets_for("archive_extract", Some(&with)).unwrap();
assert_eq!(a.reads, vec!["a.zip".to_string()]);
assert_eq!(a.produces, vec!["build/out".to_string()]);
// A pattern-only selection isn't known until the archive is opened — however
// many files match land inside `dest`, so it is a directory, not one file.
// This is what fixes round 7's regression: emptying this directory while
// keeping it present must still be answerable for drift, which only a
// directory-content hash (not an `is_dir()` presence check) can do.
assert_eq!(
a.kinds.get("build/out"),
Some(&crate::asset_kind::AssetKind::Directory)
);
}
#[test]
fn archive_extract_rejects_bad_config() {
// neither `members` nor `pattern` — nothing to select
let bad: Value = serde_yaml::from_str("archive: a.zip\ndest: out").unwrap();
assert!(assets_for("archive_extract", Some(&bad)).is_err());
// unknown field (deny_unknown_fields)
let bad: Value =
serde_yaml::from_str("archive: a.zip\nmembers: [x]\ndest: out\nbogus: 1").unwrap();
assert!(assets_for("archive_extract", Some(&bad)).is_err());
// invalid `pattern` regex
let bad: Value = serde_yaml::from_str("archive: a.zip\npattern: '('\ndest: out").unwrap();
assert!(assets_for("archive_extract", Some(&bad)).is_err());
}
#[test]
fn safe_relative_guards_zip_slip_and_preserves_case() {
// Rejected: traversal, absolute, Windows drive / UNC.
assert!(safe_relative("../evil.txt").is_none());
assert!(safe_relative("a/../../b").is_none());
assert!(safe_relative("/etc/passwd").is_none());
assert!(safe_relative("C:\\Windows\\x").is_none());
assert!(safe_relative("\\\\server\\share").is_none());
// Accepted: real case preserved, `.` components dropped.
assert_eq!(
safe_relative("REGISTRANT.tsv"),
Some(PathBuf::from("REGISTRANT.tsv"))
);
assert_eq!(
safe_relative("Sub/Dir/File.TSV"),
Some(PathBuf::from("Sub/Dir/File.TSV"))
);
assert_eq!(safe_relative("./a/./b"), Some(PathBuf::from("a/b")));
}
/// Build a DEFLATE-compressed in-memory zip of `(name, bytes)` members. Exercises
/// the `deflate-flate2` backend end-to-end (the compression the N-CEN zips use).
fn build_zip(entries: &[(&str, &[u8])], method: zip::CompressionMethod) -> Vec<u8> {
use std::io::{Cursor, Write};
let mut buf = Vec::new();
{
let mut zw = zip::ZipWriter::new(Cursor::new(&mut buf));
let opts = zip::write::SimpleFileOptions::default().compression_method(method);
for (name, bytes) in entries {
zw.start_file(*name, opts).unwrap();
zw.write_all(bytes).unwrap();
}
zw.finish().unwrap();
}
buf
}
#[test]
fn extract_members_roundtrip_preserves_case_and_selects() {
let dir = tempfile::tempdir().unwrap();
let zip_bytes = build_zip(
&[
("REGISTRANT.tsv", b"cik\tlei\n1\tABC\n"),
("FUND_REPORTED_INFO.tsv", b"series_id\tlei\nS1\tXYZ\n"),
("SUBMISSION.tsv", b"unwanted\n"),
],
zip::CompressionMethod::Deflated,
);
let members = vec![
"REGISTRANT.tsv".to_string(),
"FUND_REPORTED_INFO.tsv".to_string(),
];
let written =
extract_members(std::io::Cursor::new(&zip_bytes), dir.path(), &members, None).unwrap();
assert_eq!(written.len(), 2);
// Real (upper) case preserved on disk; content round-trips through DEFLATE.
let reg = dir.path().join("REGISTRANT.tsv");
assert!(
reg.exists(),
"REGISTRANT.tsv (real case) should exist on disk"
);
assert_eq!(std::fs::read_to_string(®).unwrap(), "cik\tlei\n1\tABC\n");
assert!(dir.path().join("FUND_REPORTED_INFO.tsv").exists());
// Non-selected member is not extracted; no leftover `.part` temp file.
assert!(!dir.path().join("SUBMISSION.tsv").exists());
assert!(!dir.path().join("REGISTRANT.tsv.part").exists());
}
#[test]
fn extract_members_selects_by_pattern() {
let dir = tempfile::tempdir().unwrap();
let zip_bytes = build_zip(
&[
("REGISTRANT.tsv", b"a"),
("FUND_REPORTED_INFO.tsv", b"b"),
("readme.txt", b"c"),
],
zip::CompressionMethod::Deflated,
);
let re = regex::Regex::new(r"\.tsv$").unwrap();
let written =
extract_members(std::io::Cursor::new(&zip_bytes), dir.path(), &[], Some(&re)).unwrap();
assert_eq!(written.len(), 2);
assert!(dir.path().join("REGISTRANT.tsv").exists());
assert!(!dir.path().join("readme.txt").exists());
}
#[test]
fn extract_members_blocks_zip_slip_write() {
let dir = tempfile::tempdir().unwrap();
// A hostile member named `../evil.txt`.
let zip_bytes = build_zip(&[("../evil.txt", b"pwned")], zip::CompressionMethod::Stored);
let re = regex::Regex::new("evil").unwrap();
let res = extract_members(std::io::Cursor::new(&zip_bytes), dir.path(), &[], Some(&re));
// Invariant: whether the writer stored `../evil.txt` verbatim (→ guard rejects,
// `Err`) or normalised it (→ lands safely inside `dest`), nothing is ever
// written to the traversal target outside `dest`.
let escaped = dir.path().parent().unwrap().join("evil.txt");
assert!(
!escaped.exists(),
"zip-slip must never write outside dest (res_ok={})",
res.is_ok()
);
}
#[cfg(feature = "opendal")]
#[test]
fn opendal_fetch_declares_only_its_output() {
let of: Value =
serde_yaml::from_str("from: s3://securelake/edgar.parquet\nto: build/edgar.parquet")
.unwrap();
let a = assets_for("opendal_fetch", Some(&of)).unwrap();
assert_eq!(a.produces, vec!["build/edgar.parquet".to_string()]);
assert!(a.reads.is_empty());
}
#[cfg(feature = "opendal")]
#[test]
fn opendal_fetch_accepts_headers() {
// `headers` is a valid config field on opendal_fetch — it is carried to
// the http(s) backend via a custom ReqwestTransport (see run()), not
// rejected. The live transport wiring is proven by the SEC fetch in the
// spike run; here we just assert the config is accepted + lineage holds.
let with: Value = serde_yaml::from_str(
"from: https://www.sec.gov/files/company_tickers.json\nto: build/tickers.json\nheaders:\n User-Agent: 'Meridian (research@meridian.online)'",
)
.unwrap();
let assets = assets_for("opendal_fetch", Some(&with)).unwrap();
assert_eq!(assets.produces, vec!["build/tickers.json".to_string()]);
assert!(assets.reads.is_empty());
}
#[cfg(feature = "http-fetch")]
#[test]
fn sha256_validator_accepts_only_a_bare_64_hex_digest() {
let digest = "6b552ea48424648dc86d00df276f93fdfc55e9ad342ce3e4affc23a3a370792b";
assert_eq!(
sha256_validator(&format!("\"{}\"", digest)),
Some(digest.to_string())
);
assert_eq!(
sha256_validator(&format!("W/\"{}\"", digest.to_ascii_uppercase())),
Some(digest.to_string())
);
// A git blob sha1 (40 hex), an opaque storage id, and an md5 are not it.
assert_eq!(
sha256_validator("\"9bb295ddab0e05d785b879661af7260fed5140fc\""),
None
);
assert_eq!(sha256_validator("\"storage-object-id\""), None);
assert_eq!(
sha256_validator("\"5d41402abc4b2a76b9719d911017c592\""),
None
);
}
#[cfg(feature = "http-fetch")]
#[test]
fn join_location_resolves_absolute_and_relative_targets() {
let base = "https://example.test/datasets/x/resolve/main/a.parquet";
assert_eq!(
join_location(base, "https://cdn.example.test/blob?sig=1").as_deref(),
Some("https://cdn.example.test/blob?sig=1")
);
assert_eq!(
join_location(base, "/blob").as_deref(),
Some("https://example.test/blob")
);
assert_eq!(
join_location(base, "b.parquet").as_deref(),
Some("https://example.test/datasets/x/resolve/main/b.parquet")
);
assert_eq!(join_location("not a url", "/blob"), None);
}
// ── the redirect fixture ────────────────────────────────────────────────
//
// A loopback origin with a dataset host's shape: `/file` answers `302` with a
// RELATIVE `Location`, and the redirect target `/blob` answers `200` under
// `STORAGE_VALIDATOR` and `304` to a conditional request carrying it. What the
// `302` itself carries is per-test, which is the axis the fetch has to get right.
// It records every request head it receives and every payload byte it writes,
// which is what the assertions below read.
#[cfg(feature = "http-fetch")]
mod origin {
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::{Arc, Mutex};
pub const PAYLOAD: &[u8] = b"the artifact bytes, transferred at most once";
pub const STORAGE_VALIDATOR: &str = "\"storage-object-id\"";
/// The headers the `302` from `/file` carries, and whether `/file` itself
/// honours a conditional request.
#[derive(Default)]
pub struct Spec {
/// Served as `X-Linked-ETag` on the redirect when set.
pub linked: Option<String>,
/// Served as `Last-Modified` on the redirect when set.
pub redirect_last_modified: Option<String>,
/// When true, `/file` answers `304` to an `If-None-Match` matching
/// `linked`; when false it always redirects.
pub honour_conditional: bool,
}
impl Spec {
/// The dataset-host shape: a `302` carrying `linked`, answering `304` to
/// a conditional request that replays it.
pub fn linked(validator: &str) -> Self {
Self {
linked: Some(validator.to_string()),
honour_conditional: true,
..Self::default()
}
}
/// The same `302`, from an origin that ignores conditional requests.
pub fn linked_unconditional(validator: &str) -> Self {
Self {
linked: Some(validator.to_string()),
..Self::default()
}
}
}
#[derive(Default)]
pub struct Log {
/// Request-target of each request, in arrival order.
pub paths: Vec<String>,
/// Full request head of each request, in arrival order.
pub heads: Vec<String>,
/// Payload bytes written to clients.
pub bytes_served: usize,
}
pub struct Origin {
port: u16,
log: Arc<Mutex<Log>>,
}
impl Origin {
pub fn start(spec: Spec) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let log = Arc::new(Mutex::new(Log::default()));
let served = Arc::clone(&log);
std::thread::spawn(move || {
for stream in listener.incoming().flatten() {
serve(stream, &served, &spec);
}
});
Self { port, log }
}
pub fn url(&self, path: &str) -> String {
format!("http://127.0.0.1:{}{}", self.port, path)
}
pub fn hits(&self, path: &str) -> usize {
self.log
.lock()
.unwrap()
.paths
.iter()
.filter(|p| *p == path)
.count()
}
pub fn bytes_served(&self) -> usize {
self.log.lock().unwrap().bytes_served
}
pub fn heads(&self) -> Vec<String> {
self.log.lock().unwrap().heads.clone()
}
}
/// True when the request head replays `validator` in `If-None-Match`.
fn replays(head: &str, validator: &str) -> bool {
head.to_ascii_lowercase().contains(&format!(
"if-none-match: {}",
validator.to_ascii_lowercase()
))
}
fn serve(mut stream: TcpStream, log: &Arc<Mutex<Log>>, spec: &Spec) {
let mut head = Vec::new();
let mut byte = [0u8; 1];
while !head.ends_with(b"\r\n\r\n") {
match stream.read(&mut byte) {
Ok(1) => head.push(byte[0]),
_ => return,
}
}
let head = String::from_utf8_lossy(&head).into_owned();
let path = head.split_whitespace().nth(1).unwrap_or("").to_string();
let origin_304 = spec.honour_conditional
&& spec.linked.as_deref().is_some_and(|v| replays(&head, v));
let target_304 = replays(&head, STORAGE_VALIDATOR);
let mut l = log.lock().unwrap();
l.paths.push(path.clone());
l.heads.push(head);
let serve_body = path == "/blob" && !target_304;
let response = if path == "/blob" {
if target_304 {
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n".to_string()
} else {
l.bytes_served += PAYLOAD.len();
format!(
"HTTP/1.1 200 OK\r\nETag: {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
STORAGE_VALIDATOR,
PAYLOAD.len()
)
}
} else if origin_304 {
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n".to_string()
} else {
let mut r = "HTTP/1.1 302 Found\r\nLocation: /blob\r\n".to_string();
if let Some(ref v) = spec.linked {
r.push_str(&format!("X-Linked-ETag: {}\r\n", v));
}
if let Some(ref lm) = spec.redirect_last_modified {
r.push_str(&format!("Last-Modified: {}\r\n", lm));
}
r.push_str("Content-Length: 0\r\nConnection: close\r\n\r\n");
r
};
drop(l);
let _ = stream.write_all(response.as_bytes());
if serve_body {
let _ = stream.write_all(PAYLOAD);
}
let _ = stream.flush();
}
}
#[cfg(feature = "http-fetch")]
fn payload_digest() -> String {
use sha2::{Digest, Sha256};
format!("{:x}", Sha256::digest(origin::PAYLOAD))
}
#[cfg(feature = "http-fetch")]
fn fetch(dir: &Path, url: &str) -> Result<StepOutput> {
let env = HashMap::new();
let with: Value =
serde_yaml::from_str(&format!("url: {}\nout: build/artifact.bin", url)).unwrap();
HttpFetch.run(&with, &test_ctx(dir, &env))
}
// The sidecar records the validator the REDIRECT offered, not the one
// the redirect target answered under. Reverting to the final hop's `ETag` puts
// `STORAGE_VALIDATOR` in the sidecar and reddens this.
#[cfg(feature = "http-fetch")]
#[test]
fn fetch_records_the_redirects_validator_not_the_final_hops() {
let digest = payload_digest();
let linked = format!("\"{}\"", digest);
let server = origin::Origin::start(origin::Spec::linked(&linked));
let dir = tempfile::tempdir().unwrap();
fetch(dir.path(), &server.url("/file")).unwrap();
let out = dir.path().join("build/artifact.bin");
assert_eq!(std::fs::read(&out).unwrap(), origin::PAYLOAD);
let meta = crate::ingress_meta::read(&out).expect("sidecar written");
assert_eq!(meta.etag.as_deref(), Some(linked.as_str()));
assert_ne!(meta.etag.as_deref(), Some(origin::STORAGE_VALIDATOR));
assert_eq!(meta.content_sha256.as_deref(), Some(digest.as_str()));
}
// A second run against an unchanged remote transfers no payload — the
// origin's byte counter does not move, and the redirect target is not asked
// for at all.
#[cfg(feature = "http-fetch")]
#[test]
fn second_fetch_against_an_unchanged_remote_transfers_no_payload() {
let digest = payload_digest();
let linked = format!("\"{}\"", digest);
let server = origin::Origin::start(origin::Spec::linked(&linked));
let dir = tempfile::tempdir().unwrap();
let url = server.url("/file");
fetch(dir.path(), &url).unwrap();
let after_first = server.bytes_served();
assert_eq!(after_first, origin::PAYLOAD.len());
fetch(dir.path(), &url).unwrap();
assert_eq!(server.bytes_served(), after_first, "second run transferred");
assert_eq!(server.hits("/blob"), 1, "redirect target re-requested");
let sent = server.heads();
let conditional = sent.last().expect("a second request was made");
assert!(
conditional
.to_ascii_lowercase()
.contains(&format!("if-none-match: {}", linked.to_ascii_lowercase())),
"{}",
conditional
);
}
// The same no-retransfer guarantee for the other redirect shape — a `302` carrying no `ETag`, no
// `Last-Modified` and no `X-Linked-ETag`, which is what an http→https upgrade or
// a release redirect sends. Nothing on the first hop is storable, so the second
// run's `304` can only come from the target's validator, reached by forwarding
// the conditional. Recording the first hop's alone re-transfers PAYLOAD.
#[cfg(feature = "http-fetch")]
#[test]
fn a_redirect_carrying_no_validator_records_the_targets() {
let server = origin::Origin::start(origin::Spec::default());
let dir = tempfile::tempdir().unwrap();
let url = server.url("/file");
let out = dir.path().join("build/artifact.bin");
fetch(dir.path(), &url).unwrap();
let after_first = server.bytes_served();
assert_eq!(after_first, origin::PAYLOAD.len());
let meta = crate::ingress_meta::read(&out).expect("sidecar written");
assert_eq!(meta.etag.as_deref(), Some(origin::STORAGE_VALIDATOR));
fetch(dir.path(), &url).unwrap();
assert_eq!(server.bytes_served(), after_first, "second run transferred");
assert_eq!(std::fs::read(&out).unwrap(), origin::PAYLOAD);
let sent = server.heads();
let to_target = sent
.last()
.expect("the target was asked")
.to_ascii_lowercase();
assert!(
to_target.contains(&format!(
"if-none-match: {}",
origin::STORAGE_VALIDATOR.to_ascii_lowercase()
)),
"{}",
to_target
);
}
// The fallback is reached only where the first hop offers nothing: a `302` with a
// `Last-Modified` and no `ETag` keeps that, and `STORAGE_VALIDATOR` — which the
// origin would not recognise — stays out of the sidecar.
#[cfg(feature = "http-fetch")]
#[test]
fn a_redirect_offering_only_last_modified_keeps_it() {
const WHEN: &str = "Wed, 21 Oct 2026 07:28:00 GMT";
let server = origin::Origin::start(origin::Spec {
redirect_last_modified: Some(WHEN.to_string()),
..Default::default()
});
let dir = tempfile::tempdir().unwrap();
let out = dir.path().join("build/artifact.bin");
fetch(dir.path(), &server.url("/file")).unwrap();
let meta = crate::ingress_meta::read(&out).expect("sidecar written");
assert_eq!(meta.last_modified.as_deref(), Some(WHEN));
assert!(meta.etag.is_none(), "{:?}", meta.etag);
}
// Where the origin declares a content hash on the redirect, the sidecar
// records it with no body transferred — here on a sidecar that predates the
// field, against an origin that ignores conditional requests entirely.
#[cfg(feature = "http-fetch")]
#[test]
fn declared_content_hash_is_recorded_without_downloading_the_body() {
let digest = payload_digest();
let server = origin::Origin::start(origin::Spec::linked_unconditional(&format!(
"\"{}\"",
digest
)));
let dir = tempfile::tempdir().unwrap();
let url = server.url("/file");
let out = dir.path().join("build/artifact.bin");
fetch(dir.path(), &url).unwrap();
let after_first = server.bytes_served();
// Roll the sidecar back to the shape a fetch wrote before `content_sha256`
// existed: the bytes' own hash, and no declaration.
let mut aged = crate::ingress_meta::read(&out).unwrap();
aged.content_sha256 = None;
crate::ingress_meta::write(&out, &aged).unwrap();
fetch(dir.path(), &url).unwrap();
assert_eq!(server.bytes_served(), after_first, "second run transferred");
assert_eq!(server.hits("/blob"), 1, "redirect target re-requested");
assert_eq!(std::fs::read(&out).unwrap(), origin::PAYLOAD);
let meta = crate::ingress_meta::read(&out).unwrap();
assert_eq!(meta.content_sha256.as_deref(), Some(digest.as_str()));
}
// The declared hash is a match test, not a blanket skip: a different one is
// fetched.
#[cfg(feature = "http-fetch")]
#[test]
fn a_changed_declared_content_hash_transfers_again() {
let digest = payload_digest();
let server = origin::Origin::start(origin::Spec::linked_unconditional(&format!(
"\"{}\"",
digest
)));
let dir = tempfile::tempdir().unwrap();
let url = server.url("/file");
let out = dir.path().join("build/artifact.bin");
fetch(dir.path(), &url).unwrap();
let mut moved = crate::ingress_meta::read(&out).unwrap();
moved.sha256 = "0".repeat(64);
moved.content_sha256 = Some("1".repeat(64));
crate::ingress_meta::write(&out, &moved).unwrap();
fetch(dir.path(), &url).unwrap();
assert_eq!(server.hits("/blob"), 2);
assert_eq!(server.bytes_served(), origin::PAYLOAD.len() * 2);
}
// A credential the Protocol set reaches the origin and stops there — the
// policy ureq applies when it follows a redirect itself.
#[cfg(feature = "http-fetch")]
#[test]
fn credentials_are_not_forwarded_to_the_redirect_target() {
let server = origin::Origin::start(origin::Spec::linked_unconditional("\"opaque\""));
let dir = tempfile::tempdir().unwrap();
let env = HashMap::new();
let with: Value = serde_yaml::from_str(&format!(
"url: {}\nout: build/artifact.bin\nheaders:\n Authorization: 'Bearer hunter2'\n Cookie: 'session=abc'",
server.url("/file")
))
.unwrap();
HttpFetch.run(&with, &test_ctx(dir.path(), &env)).unwrap();
let heads = server.heads();
let to_origin = heads.first().unwrap().to_ascii_lowercase();
let to_target = heads.last().unwrap().to_ascii_lowercase();
assert!(to_origin.contains("authorization: bearer hunter2"));
assert!(to_origin.contains("cookie: session=abc"));
assert!(!to_target.contains("authorization"), "{}", to_target);
assert!(!to_target.contains("cookie:"), "{}", to_target);
}
// ── the shared fetch cache ──────────────────────────────────────────────
//
// The fixture above is a whole origin, so these read the same counters the
// freshness tests do: `bytes_served` is the number that has to stop growing when a
// second Protocol wants bytes a first one already has.
/// An opaque validator — no content hash inside it, so a `304` is the only way the
/// origin can decline to re-send the payload. That isolates the conditional
/// request; the declared-hash shortcut has its own test below.
#[cfg(feature = "http-fetch")]
const OPAQUE: &str = "\"opaque-v1\"";
/// A cache under a directory of its own, returned with the directory so the
/// caller keeps it alive.
#[cfg(feature = "http-fetch")]
fn shared_cache() -> (tempfile::TempDir, crate::fetch_cache::FetchCache) {
let store = tempfile::tempdir().unwrap();
let cache = crate::fetch_cache::FetchCache::at(store.path().join("cache"));
(store, cache)
}
/// One Protocol's fetch of `url`, with whatever cache the run has and any extra
/// `with:` lines the test needs.
#[cfg(feature = "http-fetch")]
fn fetch_with(
dir: &Path,
url: &str,
cache: Option<&crate::fetch_cache::FetchCache>,
extra: &str,
) -> Result<StepOutput> {
let env = HashMap::new();
let with: Value =
serde_yaml::from_str(&format!("url: {url}\nout: build/artifact.bin\n{extra}")).unwrap();
HttpFetch.run(
&with,
&OpContext {
dir,
db_path: dir,
env: &env,
timeout: None,
cache,
},
)
}
#[cfg(feature = "http-fetch")]
fn artifact(dir: &Path) -> PathBuf {
dir.join("build/artifact.bin")
}
/// The single object a cache rooted at `root` holds. Reached through the
/// filesystem rather than through the cache's own accessors, because a test that
/// corrupts an entry has to reach it the way rot would.
#[cfg(feature = "http-fetch")]
fn sole_object(root: &Path) -> PathBuf {
let mut objects: Vec<_> = std::fs::read_dir(root.join("objects"))
.expect("the cache filed an object")
.map(|e| e.unwrap().path())
.collect();
assert_eq!(objects.len(), 1, "{:?}", objects);
objects.pop().unwrap()
}
/// The single locator a cache rooted at `root` holds, and the record inside it.
/// Read through the filesystem rather than through [`FetchCache::lookup`], because
/// the question is which URL the store was filed under — not whether a URL the test
/// already has in hand is present.
#[cfg(feature = "http-fetch")]
fn sole_locator(root: &Path) -> (PathBuf, crate::ingress_meta::FetchMeta) {
let mut locators: Vec<_> = std::fs::read_dir(root.join("urls"))
.expect("the cache filed a locator")
.map(|e| e.unwrap().path())
.collect();
assert_eq!(locators.len(), 1, "{:?}", locators);
let path = locators.pop().unwrap();
let record = std::fs::read_to_string(&path).unwrap();
(path, serde_yaml::from_str(&record).unwrap())
}
/// A Protocol already holding `bytes`, beside a sidecar describing them as what
/// `url` served. Every field of a `<out>.arcmeta` is a field of a file inside the
/// Protocol, so `url` is as authored as `sha256` is.
#[cfg(feature = "http-fetch")]
fn protocol_claiming(dir: &Path, url: &str, bytes: &[u8]) {
let out = artifact(dir);
std::fs::create_dir_all(out.parent().unwrap()).unwrap();
std::fs::write(&out, bytes).unwrap();
crate::ingress_meta::write(
&out,
&crate::ingress_meta::FetchMeta {
url: url.to_string(),
etag: Some(OPAQUE.to_string()),
sha256: crate::state::content_hash(bytes),
..Default::default()
},
)
.unwrap();
}
/// Two Protocols agree: same artifact bytes, and the same content identity beside
/// them. `fetched_unix` is a wall clock — no two runs share one — so it is
/// compared for presence, and every other field for value.
#[cfg(feature = "http-fetch")]
fn assert_same_result(left: &Path, right: &Path) {
let (a, b) = (artifact(left), artifact(right));
assert_eq!(
std::fs::read(&a).unwrap(),
std::fs::read(&b).unwrap(),
"artifact bytes differ between {} and {}",
left.display(),
right.display()
);
let (ma, mb) = (
crate::ingress_meta::read(&a).expect("sidecar written"),
crate::ingress_meta::read(&b).expect("sidecar written"),
);
assert_eq!(ma.url, mb.url);
assert_eq!(ma.request_headers, mb.request_headers);
assert_eq!(ma.etag, mb.etag);
assert_eq!(ma.last_modified, mb.last_modified);
assert_eq!(ma.sha256, mb.sha256);
assert_eq!(ma.content_sha256, mb.content_sha256);
assert!(ma.fetched_unix.is_some() && mb.fetched_unix.is_some());
}
// The whole point, in one test: two Protocols name the same URL and the payload
// crosses the wire once. The second still asks the origin — it asks with a validator, so
// the answer is `304` and the bytes come off the shared store.
#[cfg(feature = "http-fetch")]
#[test]
fn two_protocols_fetching_the_same_url_transfer_it_once() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let first = tempfile::tempdir().unwrap();
let second = tempfile::tempdir().unwrap();
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
server.bytes_served(),
origin::PAYLOAD.len(),
"the first Protocol transfers the artifact"
);
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
server.bytes_served(),
origin::PAYLOAD.len(),
"the second Protocol transferred it again"
);
assert_eq!(
std::fs::read(artifact(second.path())).unwrap(),
origin::PAYLOAD
);
assert_same_result(first.path(), second.path());
}
// The same, through the origin that declares a content hash and ignores
// conditional requests: the declaration is checked against what the shared store
// holds, so the body is never read even though the response is a `200`.
#[cfg(feature = "http-fetch")]
#[test]
fn a_declared_content_hash_the_cache_already_holds_transfers_nothing() {
let digest = payload_digest();
let server =
origin::Origin::start(origin::Spec::linked_unconditional(&format!("\"{digest}\"")));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let first = tempfile::tempdir().unwrap();
let second = tempfile::tempdir().unwrap();
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(server.bytes_served(), origin::PAYLOAD.len());
assert_eq!(server.hits("/blob"), 1, "the payload host was asked twice");
assert_same_result(first.path(), second.path());
}
// Parity, demonstrated: the same two Protocols against the same origin, once with
// no cache and once with one. The cached pair transfers the payload once and the
// uncached pair twice — and all four Protocols end holding the same artifact and
// the same content identity.
#[cfg(feature = "http-fetch")]
#[test]
fn a_cached_run_and_an_uncached_run_agree() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let uncached = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()];
for dir in &uncached {
fetch_with(dir.path(), &url, None, "").unwrap();
}
let without = server.bytes_served();
assert_eq!(
without,
origin::PAYLOAD.len() * 2,
"without a cache each Protocol transfers the artifact"
);
let (_store, cache) = shared_cache();
let cached = [tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap()];
for dir in &cached {
fetch_with(dir.path(), &url, Some(&cache), "").unwrap();
}
assert_eq!(
server.bytes_served() - without,
origin::PAYLOAD.len(),
"with one, the pair transfers it once"
);
for dir in [&uncached[1], &cached[0], &cached[1]] {
assert_same_result(uncached[0].path(), dir.path());
}
}
// The cache does not decide freshness. A hit is put to the origin, which is what
// keeps a cached run agreeing with an uncached one when the remote has moved —
// and the exception is a manifest that pinned the hash, which named the bytes and
// leaves the origin nothing to add.
#[cfg(feature = "http-fetch")]
#[test]
fn a_cache_hit_is_revalidated_unless_the_manifest_pinned_the_hash() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let first = tempfile::tempdir().unwrap();
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
let asked = server.hits("/file");
let second = tempfile::tempdir().unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
server.hits("/file"),
asked + 1,
"an unpinned hit must still be revalidated"
);
let third = tempfile::tempdir().unwrap();
fetch_with(
third.path(),
&url,
Some(&cache),
&format!("sha256: {}", payload_digest()),
)
.unwrap();
assert_eq!(
server.hits("/file"),
asked + 1,
"a pinned hit must not ask the origin"
);
assert_eq!(
std::fs::read(artifact(third.path())).unwrap(),
origin::PAYLOAD
);
assert_same_result(second.path(), third.path());
}
// Revalidation is not a formality: against an origin that ignores conditional
// requests, the cached run downloads exactly as the uncached run does. The store
// never answers on behalf of an origin that did not confirm it.
#[cfg(feature = "http-fetch")]
#[test]
fn an_origin_that_ignores_conditionals_is_still_the_authority() {
let server = origin::Origin::start(origin::Spec::linked_unconditional(OPAQUE));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let first = tempfile::tempdir().unwrap();
let second = tempfile::tempdir().unwrap();
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
server.bytes_served(),
origin::PAYLOAD.len() * 2,
"the origin's answer, not the cache's, decides"
);
assert_same_result(first.path(), second.path());
}
// An entry is verified on the way out, so bytes that have rotted in the store are
// refused rather than handed to the next Protocol — which costs that Protocol a
// transfer, and is the only acceptable price.
#[cfg(feature = "http-fetch")]
#[test]
fn a_corrupt_entry_is_not_served_to_a_second_consumer() {
let store = tempfile::tempdir().unwrap();
let root = store.path().join("cache");
let cache = crate::fetch_cache::FetchCache::at(root.clone());
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let first = tempfile::tempdir().unwrap();
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
// One byte flipped, length unchanged — the mutation a size check would miss.
let object = sole_object(&root);
let mut rotted = std::fs::read(&object).unwrap();
*rotted.last_mut().unwrap() ^= 0b0000_0001;
std::fs::write(&object, &rotted).unwrap();
let second = tempfile::tempdir().unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
std::fs::read(artifact(second.path())).unwrap(),
origin::PAYLOAD,
"the second Protocol got the artifact, not the rot"
);
assert_eq!(
server.bytes_served(),
origin::PAYLOAD.len() * 2,
"which cost a transfer, as it must"
);
// The rot is gone: refused, evicted, and replaced by what the transfer
// brought back — so the store heals rather than staying poisoned.
assert_eq!(
std::fs::read(sole_object(&root)).unwrap(),
origin::PAYLOAD,
"the object filed under the key is not the bytes that failed it"
);
}
// A `.arcmeta` is a file inside a Protocol, so its `sha256` is chosen by whoever
// authored the tree — and that field becomes the shared store's key, which is a
// path. Run one Protocol holding a poisoned sidecar, then a second naming the same
// URL, and the second used to unlink whatever the first pointed at: a fetch-only
// Protocol, exit 0, no message. Nothing outside the store may be touched.
#[cfg(feature = "http-fetch")]
#[test]
fn a_poisoned_sidecar_cannot_make_a_second_protocol_delete_a_file() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let elsewhere = tempfile::tempdir().unwrap();
let victim = elsewhere.path().join("important.txt");
let contents = b"someone else's work";
std::fs::write(&victim, contents).unwrap();
// The first Protocol holds the artifact and a sidecar whose `sha256` is that
// file's path rather than a digest. `Path::join` replaces the path it is given
// when the argument is absolute, so this key escapes `<root>/objects` outright.
let first = tempfile::tempdir().unwrap();
let out = artifact(first.path());
std::fs::create_dir_all(out.parent().unwrap()).unwrap();
std::fs::write(&out, origin::PAYLOAD).unwrap();
crate::ingress_meta::write(
&out,
&crate::ingress_meta::FetchMeta {
url: url.clone(),
etag: Some(OPAQUE.to_string()),
sha256: victim.to_string_lossy().into_owned(),
..Default::default()
},
)
.unwrap();
// Its next run revalidates, gets a `304`, keeps its bytes — and offers the
// sidecar to the shared store, which is where the key gets in.
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
assert!(
!cache.holds(&url),
"a locator was filed under a key that is not a digest"
);
// A second Protocol names the same URL. Its lookup is what used to evict.
let second = tempfile::tempdir().unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
std::fs::read(&victim).unwrap(),
contents,
"a fetch deleted a file outside the cache root"
);
assert_eq!(
std::fs::read(artifact(second.path())).unwrap(),
origin::PAYLOAD,
"and the second Protocol still got its artifact"
);
}
// A transfer that does not match the pin is refused before it lands: the manifest
// said which artifact it wanted, and this is not it.
#[cfg(feature = "http-fetch")]
#[test]
fn a_transfer_that_misses_the_pin_is_refused() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let dir = tempfile::tempdir().unwrap();
let wrong = crate::state::content_hash(b"some other artifact entirely");
let err = fetch_with(
dir.path(),
&server.url("/file"),
None,
&format!("sha256: {wrong}"),
)
.expect_err("a pin the bytes miss must fail the step");
assert!(
matches!(err, Error::StepExecution { .. }),
"and must not be retried: {err:?}"
);
assert!(!artifact(dir.path()).exists(), "nothing lands");
assert!(
!dir.path().join("build/artifact.part").exists(),
"and no partial file is left to be read"
);
}
// A pin that cannot be a SHA-256 is a manifest error, caught where every other
// `with:` mistake is — at load, by `assets`, before a run starts.
#[cfg(feature = "http-fetch")]
#[test]
fn a_malformed_pin_is_a_load_time_manifest_error() {
let with: Value = serde_yaml::from_str(
"url: https://x.invalid/a.parquet\nout: build/a.parquet\nsha256: deadbeef",
)
.unwrap();
let err = assets_for("http_fetch", Some(&with)).expect_err("a bad pin is refused");
assert!(matches!(err, Error::ManifestValidation(_)), "{err:?}");
assert!(err.to_string().contains("64 hex characters"), "{err}");
}
// Bytes behind a credential belong to the credential, not to the URL, and this
// store is keyed by URL. So a credentialed fetch neither fills it nor reads it.
#[cfg(feature = "http-fetch")]
#[test]
fn a_credentialed_fetch_is_not_shared() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let (_store, cache) = shared_cache();
let dir = tempfile::tempdir().unwrap();
fetch_with(
dir.path(),
&url,
Some(&cache),
"headers:\n Authorization: 'Bearer hunter2'\n",
)
.unwrap();
assert_eq!(
std::fs::read(artifact(dir.path())).unwrap(),
origin::PAYLOAD
);
assert!(
!cache.holds(&url),
"a credentialed fetch must not seed the store"
);
}
// A Protocol that already holds the artifact seeds the store from it, so the cache
// arriving mid-corpus does not cost a re-download of what is already on disk.
#[cfg(feature = "http-fetch")]
#[test]
fn an_existing_artifact_seeds_the_store_on_the_next_run() {
let server = origin::Origin::start(origin::Spec::linked(OPAQUE));
let url = server.url("/file");
let first = tempfile::tempdir().unwrap();
// Fetched before there was a cache to file it in.
fetch_with(first.path(), &url, None, "").unwrap();
let (_store, cache) = shared_cache();
assert!(!cache.holds(&url));
// The next run of the same Protocol revalidates, keeps its bytes, and files
// them.
fetch_with(first.path(), &url, Some(&cache), "").unwrap();
assert!(
cache.holds(&url),
"the store is seeded from what was already there"
);
let second = tempfile::tempdir().unwrap();
fetch_with(second.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
server.bytes_served(),
origin::PAYLOAD.len(),
"so a second Protocol transfers nothing"
);
assert_same_result(first.path(), second.path());
}
// The store files a locator under the record's own `url`, and the record a keeping
// Protocol offers it is `<out>.arcmeta` — a file inside that Protocol. So a
// Protocol whose only step fetches its OWN origin could name a third party's URL in
// the sidecar and have the store file that URL against bytes nobody fetched from
// it. Exit 0, no warning, and the next Protocol to name the URL for real is served
// them under a sidecar carrying the genuine URL and the genuine validator.
//
// Two loopback origins, because the whole defect is the gap between the URL the
// step names and the URL the sidecar does.
#[cfg(feature = "http-fetch")]
#[test]
fn a_sidecar_cannot_seed_the_store_for_a_url_the_step_did_not_name() {
const SUBSTITUTED: &[u8] = b"bytes the genuine origin never served";
let hostile_origin = origin::Origin::start(origin::Spec::linked(OPAQUE));
let genuine_origin = origin::Origin::start(origin::Spec::linked(OPAQUE));
let hostile_url = hostile_origin.url("/file");
let genuine_url = genuine_origin.url("/file");
let store = tempfile::tempdir().unwrap();
let root = store.path().join("cache");
let cache = crate::fetch_cache::FetchCache::at(root.clone());
// One `http_fetch`, of its own origin. It holds its own bytes already, so the
// conditional request is answered `304` and the keep-and-seed path runs.
let hostile = tempfile::tempdir().unwrap();
protocol_claiming(hostile.path(), &genuine_url, SUBSTITUTED);
fetch_with(hostile.path(), &hostile_url, Some(&cache), "").unwrap();
assert!(
!cache.holds(&genuine_url),
"a Protocol that never fetched that URL filed the store's locator for it"
);
assert_eq!(
sole_locator(&root).1.url,
hostile_url,
"the store may only be told about the URL the step named"
);
// What a later Protocol gets for the genuine URL, against what it gets with no
// store at all — the parity a cached run owes an uncached one, which is the
// acceptance this defect fails as well as the substitution itself.
let served = tempfile::tempdir().unwrap();
fetch_with(served.path(), &genuine_url, Some(&cache), "").unwrap();
assert_eq!(
std::fs::read(artifact(served.path())).unwrap(),
origin::PAYLOAD,
"a later Protocol was served the substituted bytes"
);
let uncached = tempfile::tempdir().unwrap();
fetch_with(uncached.path(), &genuine_url, None, "").unwrap();
assert_same_result(served.path(), uncached.path());
}
// The declared-content-hash shortcut decides on `sha256`, which the store has
// hashed against the object's bytes, and not on `content_sha256` — a value copied
// from a response head into a `.arcmeta`, which is plain YAML inside a Protocol
// directory that anything able to write a file can author.
//
// The two differ when the verified hash says the bytes are not the declared
// artifact, so a shortcut that accepts the claim keeps bytes it has just been told
// are the wrong ones — here in both directions. A Protocol holding substituted
// bytes answers the origin's declaration with a copy of the declaration, skips the
// transfer, keeps the substitution and files it in the shared store for the URL its
// step named; the next Protocol to name that URL is handed the same match and
// served the substituted bytes.
//
// Neither half needs a `304` or a live validator: the sidecar's `ETag` here is
// worthless and this origin ignores conditional requests, so the shortcut is the
// whole of what decides.
#[cfg(feature = "http-fetch")]
#[test]
fn a_declared_content_hash_is_matched_against_verified_bytes_only() {
const SUBSTITUTED: &[u8] = b"bytes the genuine origin never served";
let digest = payload_digest();
let server =
origin::Origin::start(origin::Spec::linked_unconditional(&format!("\"{digest}\"")));
let url = server.url("/file");
let (_store, cache) = shared_cache();
// A Protocol whose artifact is not what the origin serves, beside a sidecar
// declaring the origin's own published content hash for it.
let seeding = tempfile::tempdir().unwrap();
let out = artifact(seeding.path());
std::fs::create_dir_all(out.parent().unwrap()).unwrap();
std::fs::write(&out, SUBSTITUTED).unwrap();
crate::ingress_meta::write(
&out,
&crate::ingress_meta::FetchMeta {
url: url.clone(),
etag: Some("\"stale-and-worthless\"".to_string()),
sha256: crate::state::content_hash(SUBSTITUTED),
content_sha256: Some(digest.clone()),
..Default::default()
},
)
.unwrap();
fetch_with(seeding.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
std::fs::read(&out).unwrap(),
origin::PAYLOAD,
"the transfer was skipped on a claim the sidecar made about itself"
);
assert_eq!(
cache.lookup(&url).map(|e| e.sha256),
Some(digest.clone()),
"and the shared store was seeded with bytes the origin never served"
);
// What a second Protocol naming the same URL is served, against what it gets
// with no store at all.
let served = tempfile::tempdir().unwrap();
fetch_with(served.path(), &url, Some(&cache), "").unwrap();
assert_eq!(
std::fs::read(artifact(served.path())).unwrap(),
origin::PAYLOAD,
"a second Protocol was served the substituted bytes"
);
let uncached = tempfile::tempdir().unwrap();
fetch_with(uncached.path(), &url, None, "").unwrap();
assert_same_result(served.path(), uncached.path());
}
// The guard that decides whether to seed and the write it guards have to ask about
// the same URL. While the write took the sidecar's `url` and the guard asked about
// the step's, the guard's answer could never become true — so a Protocol that keeps
// its bytes re-filed the store on every single run.
#[cfg(feature = "http-fetch")]
#[test]
fn a_seeded_store_is_not_re_filed_on_every_run() {
const KEPT: &[u8] = b"bytes this Protocol already had";
let hostile_origin = origin::Origin::start(origin::Spec::linked(OPAQUE));
let genuine_origin = origin::Origin::start(origin::Spec::linked(OPAQUE));
let hostile_url = hostile_origin.url("/file");
let store = tempfile::tempdir().unwrap();
let root = store.path().join("cache");
let cache = crate::fetch_cache::FetchCache::at(root.clone());
let dir = tempfile::tempdir().unwrap();
protocol_claiming(dir.path(), &genuine_origin.url("/file"), KEPT);
fetch_with(dir.path(), &hostile_url, Some(&cache), "").unwrap();
// Stamp the entry the first run filed. A second write rewrites the record
// wholesale, so the stamp surviving is the proof the store was left alone.
let (locator, mut filed) = sole_locator(&root);
filed.etag = Some("\"filed-once\"".to_string());
std::fs::write(&locator, serde_yaml::to_string(&filed).unwrap()).unwrap();
fetch_with(dir.path(), &hostile_url, Some(&cache), "").unwrap();
assert_eq!(
sole_locator(&root).1.etag,
filed.etag,
"the store was filed again by a run that had nothing new to tell it"
);
}
// ─────────────────────────────────────────────────────────────────────────
// uv@1 — the two refusals that make it an operator rather than a `command:`,
// the dependency pin, and the argv.
//
// Each refusal below is driven from a config that is otherwise VALID, changed
// in exactly one way. That is the shape the mutation half of this work needs:
// the passing case and the refusing case differ by the one field under test,
// so a refusal that stops firing cannot be masked by a second defect.
// ─────────────────────────────────────────────────────────────────────────
/// A `with:` block that passes every check — the base each refusal mutates.
fn uv_ok() -> Value {
serde_yaml::from_str(
r#"
script: scripts/derive.py
sha256: "0000000000000000000000000000000000000000000000000000000000000000"
deps: ["duckdb==1.5.5"]
args: ["--db", "build/pipeline.duckdb"]
reads: ["build/crosswalk.parquet"]
produces: ["build/coverage.json"]
"#,
)
.unwrap()
}
/// The base config is genuinely accepted. Without this, every refusal below
/// could be firing for a reason that has nothing to do with the field it names.
#[test]
fn uv_accepts_a_fully_declared_step() {
let assets = Uv.assets(&uv_ok()).expect("the base config must be valid");
assert_eq!(assets.reads, vec!["build/crosswalk.parquet".to_string()]);
assert_eq!(assets.produces, vec!["build/coverage.json".to_string()]);
}
/// Drop `reads:` from the base config and the operator refuses, naming the
/// field. A Python step invisible to staleness is the defect `uv@1` exists to
/// close, so it must not be expressible.
#[test]
fn uv_refuses_a_step_that_declares_no_reads() {
let mut with = uv_ok();
with.as_mapping_mut().unwrap().remove(Value::from("reads"));
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(
err.contains("`reads:` declares nothing"),
"the refusal must name the field: {err}"
);
assert!(err.contains("asset graph"), "and say why it matters: {err}");
}
/// The same for `produces:`, which is the half that lets a step DOWNSTREAM of
/// this one go stale.
#[test]
fn uv_refuses_a_step_that_declares_no_produces() {
let mut with = uv_ok();
with.as_mapping_mut()
.unwrap()
.remove(Value::from("produces"));
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(
err.contains("`produces:` declares nothing"),
"the refusal must name the field: {err}"
);
}
/// `reads: []` costs exactly what an absent `reads:` costs — a step with no
/// edges — so it is refused identically. A rule that reads only for the key's
/// presence is a rule with one spelling of its own defect left open.
#[test]
fn uv_refuses_an_empty_reads_list_the_same_way_as_an_absent_one() {
for yaml in ["reads: []", r#"reads: ["", " "]"#] {
let mut with = uv_ok();
let empty: Value = serde_yaml::from_str(yaml).unwrap();
let list = empty
.as_mapping()
.unwrap()
.get(Value::from("reads"))
.unwrap()
.clone();
with.as_mapping_mut()
.unwrap()
.insert(Value::from("reads"), list);
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(
err.contains("`reads:` declares nothing"),
"`{yaml}` must be refused like an absent reads:, got: {err}"
);
}
}
/// Every spelling that leaves the resolver a choice, refused **by name** — and
/// the assertion is on the reason, not on the fact of a refusal.
///
/// The distinction is the whole test. `unpinned_dependency` is a ladder of
/// guards, and deleting any rung leaves a lower one catching the same input for a
/// different reason: strike `s.contains(';')` and an environment marker is caught
/// by the second-constraint rung instead, because `python_version < '3.13'`
/// carries a `<`. A test asserting only that the entry was refused is green
/// against every one of those deletions. The mutation gate found exactly that,
/// four rungs at once, and this is the repair — each row names the rung that is
/// supposed to catch it.
///
/// The property is about time, not style: a range resolves to whatever the index
/// holds on the day it runs, so two Runs a month apart install different code
/// while the manifest, the digest and the asset graph all read as unchanged.
#[test]
fn uv_refuses_every_unpinned_dependency_spelling_for_the_right_reason() {
for (spec, rung) in [
("", "names no package"),
(
"duckdb==1.5.5; python_version < '3.13'",
"environment marker",
),
(
"duckdb @ https://example.invalid/duckdb.whl",
"direct reference",
),
("duckdb", "carries no `==`"),
("duckdb>=1.5.5", "carries no `==`"),
("duckdb<2", "carries no `==`"),
("duckdb~=1.5.5", "carries no `==`"),
("duckdb!=1.5.4", "carries no `==`"),
("duckdb>=1.5,<2", "carries no `==`"),
("duckdb==1.5.5,!=1.5.4", "exactly one `==`"),
("duckdb===1.5.5", "exactly one `==`"),
("duckdb[==1.5.5", "never closed"),
("duckdb[]==1.5.5", "not a list of extras"),
("duckdb[a b]==1.5.5", "not a list of extras"),
("[extras]==1.5.5", "not a distribution name"),
("duck db==1.5.5", "not a distribution name"),
("duckdb==", "nothing after it"),
("duckdb==1.5.*", "wildcard"),
("duckdb==1!5.5", "not a version"),
] {
let why = unpinned_dependency(spec)
.unwrap_or_else(|| panic!("`{spec}` leaves the version open and must be refused"));
assert!(
why.contains(rung),
"`{spec}` must be refused for `{rung}`, not for: {why}"
);
// And the refusal reaches a manifest author with the entry quoted.
let mut with = uv_ok();
with.as_mapping_mut().unwrap().insert(
Value::from("deps"),
Value::Sequence(vec![Value::from(spec)]),
);
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(err.contains(rung), "the reason reaches the author: {err}");
assert!(
err.contains("package==version"),
"and so does the pinned form: {err}"
);
}
}
/// The other direction, which matters as much: a rule that bites an honest pin
/// is worse than the gap it closes. Extras, epochs-free local versions and
/// pre-releases are all legitimate pins.
#[test]
fn uv_accepts_the_pinned_spellings() {
for spec in [
"duckdb==1.5.5",
"pandas==2.2.3",
"tomli_w==1.2.0",
"tomli-w==1.2.0",
"pydantic[email]==2.9.2",
"torch==2.4.0+cpu",
"numpy==2.0.0rc1",
" duckdb==1.5.5 ",
] {
assert_eq!(
unpinned_dependency(spec),
None,
"`{spec}` is a pin and must be accepted"
);
}
}
/// A blank `script:` names no file, so it is refused rather than turned into a
/// `dir.join("")` that resolves to the protocol directory itself.
#[test]
fn uv_refuses_a_blank_script() {
for spelling in ["", " "] {
let mut with = uv_ok();
with.as_mapping_mut()
.unwrap()
.insert(Value::from("script"), Value::from(spelling));
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(
err.contains("`script:` is blank"),
"a blank script must be refused, got: {err}"
);
}
}
/// `sha256:` is compared as text against a lowercase hex digest, so anything
/// that is not one is refused at load rather than mismatching at run.
#[test]
fn uv_refuses_a_sha256_that_is_not_lowercase_hex() {
for (digest, expect) in [
("deadbeef", "64 hex characters"),
(
"00000000000000000000000000000000000000000000000000000000000000zz",
"64 hex characters",
),
(
"ABCDEF0000000000000000000000000000000000000000000000000000000000",
"lowercase",
),
] {
let mut with = uv_ok();
with.as_mapping_mut()
.unwrap()
.insert(Value::from("sha256"), Value::from(digest));
let err = Uv.assets(&with).unwrap_err().to_string();
assert!(
err.contains(expect),
"`{digest}` must be refused for {expect}, got: {err}"
);
}
}
/// `run` validates again rather than trusting that manifest load did.
///
/// It is not redundant and it is not covered by the load-time tests: those call
/// `assets`, and neutering `run`'s own `cfg.validate()?` left all of them green.
/// A caller that reaches `run` without going through manifest validation — a
/// generated step, a future embedding of the engine — would otherwise spawn `uv`
/// on a config the operator has already decided is not expressible.
#[test]
fn uv_run_refuses_an_invalid_config_rather_than_spawning() {
let mut with = uv_ok();
with.as_mapping_mut().unwrap().remove(Value::from("reads"));
let tmp = tempfile::tempdir().unwrap();
let env = HashMap::new();
let err = Uv
.run(&with, &test_ctx(tmp.path(), &env))
.expect_err("run must refuse a config assets() would have refused");
assert!(
err.to_string().contains("`reads:` declares nothing"),
"and refuse it for the same reason: {err}"
);
assert!(
matches!(err, Error::ManifestValidation(_)),
"a config defect is a validation error wherever it is caught"
);
}
/// A Run whose script bytes differ from the pin refuses BEFORE `uv` is spawned,
/// and the message carries the step, the digest expected and the digest found —
/// so re-pinning is a copy-paste rather than a hunt.
#[test]
fn uv_refuses_a_script_whose_bytes_do_not_match_the_pin() {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir_all(tmp.path().join("scripts")).unwrap();
let body = b"print('hello')\n";
std::fs::write(tmp.path().join("scripts/derive.py"), body).unwrap();
let real = crate::state::content_hash(body);
let cfg = UvConfig::parse(&uv_ok()).unwrap();
let err = uv_invocation(&cfg, tmp.path()).unwrap_err();
let msg = err.to_string();
assert!(msg.contains("uv"), "names the step: {msg}");
assert!(
msg.contains(&"0".repeat(64)),
"names the digest the manifest pinned: {msg}"
);
assert!(msg.contains(&real), "names the digest found: {msg}");
// Non-retryable: a script whose bytes disagree with the manifest will
// disagree again, so a Protocol carrying `defaults.retry` must not pay its
// backoff three times to be told the same thing.
assert!(
matches!(err, Error::StepExecution { .. }),
"a digest mismatch is deterministic, so it is not a retryable failure"
);
// And with the digest corrected, the same config builds an invocation.
let mut with = uv_ok();
with.as_mapping_mut()
.unwrap()
.insert(Value::from("sha256"), Value::from(real.as_str()));
let cfg = UvConfig::parse(&with).unwrap();
uv_invocation(&cfg, tmp.path()).expect("the pin now matches the bytes");
}
/// The hole a byte digest structurally cannot close: `uv run --script` reads the
/// script's OWN PEP 723 header and installs what it names, in addition to every
/// `--with`. So a step declaring no `deps:` at all, with its script pinned
/// byte-for-byte, could resolve an unpinned package on the day it ran — because
/// the unpinned declaration is part of the bytes the digest pins.
///
/// Reproduced end to end before this test existed, with a header reading
/// `dependencies = ["iniconfig"]`: `arc run` installed one and succeeded.
#[test]
fn uv_refuses_an_unpinned_dependency_declared_inside_the_script() {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir_all(tmp.path().join("scripts")).unwrap();
let body = "# /// script\n# requires-python = \">=3.12\"\n# dependencies = [\n# \"iniconfig\",\n# ]\n# ///\nprint('hi')\n";
std::fs::write(tmp.path().join("scripts/derive.py"), body).unwrap();
let mut with = uv_ok();
with.as_mapping_mut().unwrap().insert(
Value::from("sha256"),
Value::from(crate::state::content_hash(body.as_bytes())),
);
// No `deps:` at all — the manifest is silent, which is exactly the case the
// digest was assumed to cover.
with.as_mapping_mut().unwrap().remove(Value::from("deps"));
let cfg = UvConfig::parse(&with).unwrap();
let err = uv_invocation(&cfg, tmp.path()).unwrap_err().to_string();
assert!(err.contains("iniconfig"), "names the entry: {err}");
assert!(err.contains("PEP 723"), "and where it came from: {err}");
// And a PINNED one in the same place is accepted, so the rule is about the pin
// rather than about the block.
let pinned = body.replace("\"iniconfig\"", "\"iniconfig==2.0.0\"");
std::fs::write(tmp.path().join("scripts/derive.py"), &pinned).unwrap();
let mut with = uv_ok();
with.as_mapping_mut().unwrap().insert(
Value::from("sha256"),
Value::from(crate::state::content_hash(pinned.as_bytes())),
);
with.as_mapping_mut().unwrap().remove(Value::from("deps"));
let cfg = UvConfig::parse(&with).unwrap();
uv_invocation(&cfg, tmp.path()).expect("a pinned PEP 723 dependency is fine");
}
/// A block this operator cannot read is refused, not ignored — what it cannot see,
/// `uv` installs anyway.
#[test]
fn uv_refuses_a_pep_723_block_it_cannot_read() {
for (label, body) in [
(
"never closed",
"# /// script\n# dependencies = [\n# \"x==1\",\n# ]\nprint('hi')\n",
),
(
"unterminated string",
"# /// script\n# dependencies = [\n# \"x==1,\n# ]\n# ///\nprint('hi')\n",
),
(
"list not closed",
"# /// script\n# dependencies = [\n# \"x==1\",\n# ///\nprint('hi')\n",
),
// The block runs to END OF FILE. This is the case the `!closed` branch
// exists for, and it is NOT the same as the first: there the loop meets a
// non-comment line and refuses on that, so returning an empty list at
// end-of-file was invisible until this row. The dependency here is PINNED
// on purpose, so the only thing that can refuse it is the unclosed block.
(
"closes at end of file",
"# /// script\n# dependencies = [\n# \"x==1\",\n# ]\n",
),
] {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir_all(tmp.path().join("scripts")).unwrap();
std::fs::write(tmp.path().join("scripts/derive.py"), body).unwrap();
let mut with = uv_ok();
with.as_mapping_mut().unwrap().insert(
Value::from("sha256"),
Value::from(crate::state::content_hash(body.as_bytes())),
);
let cfg = UvConfig::parse(&with).unwrap();
let err = uv_invocation(&cfg, tmp.path()).unwrap_err().to_string();
assert!(
err.contains("PEP 723"),
"`{label}` must be refused as an unreadable block, got: {err}"
);
}
}
/// The parser reads the block and only the block. Each case is a shape that would
/// let an unpinned dependency through if it were read wrongly.
#[test]
fn script_declared_dependencies_reads_the_block_and_nothing_else() {
// No block at all.
assert_eq!(
script_declared_dependencies("print('hi')\n").unwrap(),
Vec::<String>::new()
);
// A block with no dependencies key.
assert_eq!(
script_declared_dependencies("# /// script\n# requires-python = \">=3.12\"\n# ///\n")
.unwrap(),
Vec::<String>::new()
);
// The real shape, single and double quoted, across lines.
assert_eq!(
script_declared_dependencies(
"# /// script\n# dependencies = [\n# \"duckdb==1.5.5\",\n# 'numpy==2.0.0',\n# ]\n# ///\n"
)
.unwrap(),
vec!["duckdb==1.5.5".to_string(), "numpy==2.0.0".to_string()]
);
// A mention of the opener inside prose is not the block — it is not at a line
// start, so it is not read.
assert_eq!(
script_declared_dependencies("x = \"# /// script\"\n").unwrap(),
Vec::<String>::new()
);
// A key that merely starts with `dependencies` is not the list.
assert!(
script_declared_dependencies(
"# /// script\n# dependencies_note = \"see below\"\n# ///\n"
)
.is_err()
|| script_declared_dependencies(
"# /// script\n# dependencies_note = \"see below\"\n# ///\n"
)
.unwrap()
.is_empty()
);
}
/// A script that is not on disk is the same class — deterministic, so
/// non-retryable — and the message says this operator does not fetch one.
#[test]
fn uv_refuses_a_script_that_is_not_on_disk() {
let tmp = tempfile::tempdir().unwrap();
let cfg = UvConfig::parse(&uv_ok()).unwrap();
let err = uv_invocation(&cfg, tmp.path()).unwrap_err();
assert!(
matches!(err, Error::StepExecution { .. }),
"a missing script is not retryable"
);
let msg = err.to_string();
assert!(msg.contains("scripts/derive.py"), "names the script: {msg}");
// Pinned at the WORDING, not merely at the type. Letting the read error fall
// through would hash empty bytes instead, and the digest mismatch that
// followed is also a `StepExecution` that also names the script — so an
// assertion on either of those alone cannot tell a missing file from an
// edited one, and would tell the reader the wrong thing about their tree.
assert!(
msg.contains("is not readable"),
"says the file is absent rather than reporting a digest mismatch \
against empty bytes: {msg}"
);
assert!(
!msg.contains("does not match the digest"),
"a missing script is not a pin failure: {msg}"
);
}
/// Every `--with` precedes `--script`, and the script's own arguments follow
/// the path. `uv` reads `--with` as its own flag and everything after the
/// script path as the script's, so this order is not cosmetic.
#[test]
fn uv_op_args_put_every_with_before_the_script() {
assert_eq!(
uv_op_args(
"/cache/derive.py",
&["duckdb==1.5.5".to_string(), "tomli_w==1.2.0".to_string()],
&["--db".to_string(), "build/p.duckdb".to_string()],
),
vec![
"run",
"--with",
"duckdb==1.5.5",
"--with",
"tomli_w==1.2.0",
"--script",
"/cache/derive.py",
"--db",
"build/p.duckdb",
]
);
}
/// With no dependencies the argv is exactly what the frozen-script operators
/// build, so `uv@1` is the same substrate rather than a parallel one.
#[test]
fn uv_op_args_with_no_deps_match_the_frozen_script_invocation() {
assert_eq!(
uv_op_args("/cache/derive.py", &[], &["--x".to_string()]),
uv_run_args("/cache/derive.py", &["--x".to_string()])
);
}
/// The declared names carry the kind their spelling implies — the same rule a
/// manifest `depends_on:` gets, because these are strings a human wrote with
/// no parser behind them.
#[test]
fn uv_declares_the_kind_each_name_spells() {
let with: Value = serde_yaml::from_str(
r#"
script: s.py
sha256: "0000000000000000000000000000000000000000000000000000000000000000"
reads: ["build/in.parquet", "build/parts/", "build/shards/*.parquet"]
produces: ["build/out.json"]
"#,
)
.unwrap();
let assets = Uv.assets(&with).unwrap();
use crate::asset_kind::AssetKind;
assert_eq!(assets.kinds["build/in.parquet"], AssetKind::File);
assert_eq!(assets.kinds["build/parts/"], AssetKind::Directory);
assert_eq!(assets.kinds["build/shards/*.parquet"], AssetKind::Pattern);
assert_eq!(assets.kinds["build/out.json"], AssetKind::File);
}
}