set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
OPENAPI_URL="${ALPACA_MARKET_DATA_OPENAPI_URL:-https://docs.alpaca.markets/openapi/market-data-api.json}"
MANIFEST_PATH="${ALPACA_MARKET_DATA_MANIFEST_PATH:-$REPO_ROOT/tools/api-coverage/market-data-api.json}"
DOCS_PATH="${ALPACA_MARKET_DATA_DOCS_PATH:-$REPO_ROOT/docs/api-coverage.md}"
OBSERVED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
if [ "$#" -ne 0 ]; then
echo "error: ./scripts/api-sync-audit does not accept command-line arguments" >&2
exit 2
fi
require_tool() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "error: required tool '$1' is not installed" >&2
exit 2
fi
}
require_tool curl
require_tool jq
require_tool comm
require_tool sort
require_tool mktemp
require_tool date
if [ ! -f "$MANIFEST_PATH" ]; then
echo "error: missing manifest at $MANIFEST_PATH" >&2
exit 2
fi
if [ ! -f "$DOCS_PATH" ]; then
echo "error: missing docs contract at $DOCS_PATH" >&2
exit 2
fi
openapi_file="$(mktemp)"
official_paths_file="$(mktemp)"
official_adopted_paths_file="$(mktemp)"
official_adopted_tags_file="$(mktemp)"
manifest_covered_paths_file="$(mktemp)"
manifest_mirror_paths_file="$(mktemp)"
manifest_mirror_methods_file="$(mktemp)"
missing_contract_paths_file="$(mktemp)"
stale_mirror_paths_file="$(mktemp)"
missing_method_defs_file="$(mktemp)"
summary_mismatches_file="$(mktemp)"
manifest_gap_paths_file="$(mktemp)"
manifest_gap_tags_file="$(mktemp)"
official_not_adopted_tags_file="$(mktemp)"
untracked_official_tags_file="$(mktemp)"
stale_manifest_tags_file="$(mktemp)"
official_crypto_loc_file="$(mktemp)"
local_crypto_loc_file="$(mktemp)"
missing_crypto_loc_file="$(mktemp)"
parameter_drift_file="$(mktemp)"
response_drift_file="$(mktemp)"
affected_methods_file="$(mktemp)"
convenience_notes_file="$(mktemp)"
cleanup() {
rm -f \
"$openapi_file" \
"$official_paths_file" \
"$official_adopted_paths_file" \
"$official_adopted_tags_file" \
"$manifest_covered_paths_file" \
"$manifest_mirror_paths_file" \
"$manifest_mirror_methods_file" \
"$missing_contract_paths_file" \
"$stale_mirror_paths_file" \
"$missing_method_defs_file" \
"$summary_mismatches_file" \
"$manifest_gap_paths_file" \
"$manifest_gap_tags_file" \
"$official_not_adopted_tags_file" \
"$untracked_official_tags_file" \
"$stale_manifest_tags_file" \
"$official_crypto_loc_file" \
"$local_crypto_loc_file" \
"$missing_crypto_loc_file" \
"$parameter_drift_file" \
"$response_drift_file" \
"$affected_methods_file" \
"$convenience_notes_file"
}
trap cleanup EXIT
read -r -d '' JQ_LIB <<'JQ' || true
def pointer_target($root; $ref):
($ref | sub("^#/"; "") | split("/") | map(gsub("~1"; "/") | gsub("~0"; "~")))
| reduce .[] as $segment ($root; .[$segment]);
def deref($root; $node):
if ($node | type) == "object" and $node["$ref"]? != null then
deref($root; pointer_target($root; $node["$ref"]))
else
$node
end;
def merge_schema($left; $right):
($left * $right)
| if ($left.required? != null) or ($right.required? != null) then
.required = ((($left.required // []) + ($right.required // [])) | unique | sort)
else
.
end
| if ($left.properties? != null) or ($right.properties? != null) then
.properties = (($left.properties // {}) * ($right.properties // {}))
else
.
end;
def normalized_schema($root; $schema):
deref($root; $schema) as $resolved
| if ($resolved.allOf? | type) == "array" then
reduce $resolved.allOf[] as $part (($resolved | del(.allOf)); merge_schema(.; normalized_schema($root; $part)))
else
$resolved
end;
def scalar_string:
if . == null then
"null"
elif type == "string" then
.
else
tostring
end;
def type_label($schema):
if $schema == null then
"unknown"
elif ($schema.type? | type) == "array" then
($schema.type | map(tostring) | sort | join("|"))
elif $schema.type? != null then
($schema.type | tostring)
elif $schema.properties? != null or $schema.additionalProperties? != null then
"object"
elif $schema.items? != null then
"array"
elif $schema.enum? != null then
"string"
else
"unknown"
end;
def parameter_signatures($root; $path_item; $operation):
(($path_item.parameters // []) + ($operation.parameters // []))
| map(deref($root; .))
| unique_by(.name + "|" + .in)
| sort_by(.in, .name)
| map(
normalized_schema($root; .schema) as $schema
| [
.name,
.in,
(if .required then "required" else "optional" end),
type_label($schema),
(if $schema.items? != null then "items=" + type_label(normalized_schema($root; $schema.items)) else empty end),
(if $schema.enum? != null then "enum=" + ($schema.enum | map(scalar_string) | sort | join(",")) else empty end),
(if $schema.default? != null then "default=" + ($schema.default | scalar_string) else empty end)
]
| map(select(length > 0))
| join("|")
);
def field_signatures($root; $schema; $path):
normalized_schema($root; $schema) as $resolved
| if (($resolved.properties? | type) == "object") or ($resolved.additionalProperties? != null) or (type_label($resolved) == "object") then
((if ($path | length) > 0 then [{path: $path, type: "object"}] else [] end)
+ ([($resolved.properties // {}) | to_entries[]? | field_signatures($root; .value; (if ($path | length) == 0 then .key else $path + "." + .key end))] | add // [])
+ (if $resolved.additionalProperties? != null then field_signatures($root; $resolved.additionalProperties; (if ($path | length) == 0 then "*" else $path + ".*" end)) else [] end))
elif ($resolved.items? != null) or (type_label($resolved) == "array") then
((if ($path | length) > 0 then [{path: $path, type: "array"}] else [] end)
+ (if $resolved.items? != null then field_signatures($root; $resolved.items; ($path + "[]")) else [] end))
elif ($path | length) == 0 then
[]
else
[{path: $path, type: (type_label($resolved) + (if $resolved.enum? != null then "|enum=" + ($resolved.enum | map(scalar_string) | sort | join(",")) else "" end))}]
end;
def response_contract($root; $operation):
($operation.responses // {}) as $responses
| (["200", "201", "202", "203", "204", "206", "default"] | map(select($responses[.] != null)) | .[0]) as $status
| if $status == null then
{status: null, field_signatures: []}
else
($responses[$status] | deref($root; .)) as $response
| (($response.content["application/json"].schema // ($response.content | to_entries[0]? | .value.schema) // null)) as $schema
| {
status: $status,
field_signatures: (
if $schema == null then
[]
else
[field_signatures($root; $schema; "")[] | "\(.path)|\(.type)"] | unique | sort
end
)
}
end;
JQ
extract_parameter_signatures() {
local path="$1"
jq -r --arg path "$path" "$JQ_LIB
. as \$root
| (\$root.paths[\$path] // empty) as \$path_item
| (\$path_item.get // empty) as \$operation
| if \$operation == null then
empty
else
parameter_signatures(\$root; \$path_item; \$operation)[]
end
" "$openapi_file" | sort -u
}
extract_response_contract() {
local path="$1"
jq -c --arg path "$path" "$JQ_LIB
. as \$root
| (\$root.paths[\$path] // empty) as \$path_item
| (\$path_item.get // empty) as \$operation
| if \$operation == null then
{status: null, field_signatures: []}
else
response_contract(\$root; \$operation)
end
" "$openapi_file"
}
curl -fsSL "$OPENAPI_URL" >"$openapi_file"
jq -r '.paths | keys[]' "$openapi_file" | sort -u >"$official_paths_file"
jq -r '
.paths
| to_entries[]
| select((.value.get.tags // []) | any(. == "Stock" or . == "Option" or . == "Crypto" or . == "News" or . == "Corporate actions"))
| .key
' "$openapi_file" | sort -u >"$official_adopted_paths_file"
printf '%s\n' "Stock" "Option" "Crypto" "News" "Corporate actions" | sort -u >"$official_adopted_tags_file"
jq -r '
(.mirror_layer[]?.path),
(.gaps_in_adopted_families[]?.path)
' "$MANIFEST_PATH" | sort -u >"$manifest_covered_paths_file"
jq -r '.mirror_layer[]?.path' "$MANIFEST_PATH" | sort -u >"$manifest_mirror_paths_file"
jq -r '
.mirror_layer[]
| [.path, .local_resource, .mirror_method]
| @tsv
' "$MANIFEST_PATH" >"$manifest_mirror_methods_file"
jq -r '.gaps_in_adopted_families[]?.path' "$MANIFEST_PATH" | sort -u >"$manifest_gap_paths_file"
jq -r '.not_adopted_resource_families[]?.tag' "$MANIFEST_PATH" | sort -u >"$manifest_gap_tags_file"
comm -23 "$official_adopted_paths_file" "$manifest_covered_paths_file" >"$missing_contract_paths_file"
comm -23 "$manifest_mirror_paths_file" "$official_adopted_paths_file" >"$stale_mirror_paths_file"
while IFS=$'\t' read -r path resource method; do
[ -n "$path" ] || continue
[ -n "$resource" ] || continue
[ -n "$method" ] || continue
client_file="$REPO_ROOT/src/$resource/client.rs"
if [ ! -f "$client_file" ] || ! grep -F "pub async fn ${method}(" "$client_file" >/dev/null 2>&1; then
printf '%s\t%s\t%s\t%s\n' "$path" "$resource" "$method" "$client_file" >>"$missing_method_defs_file"
continue
fi
if ! jq -e --arg path "$path" '.paths[$path].get? != null' "$openapi_file" >/dev/null 2>&1; then
continue
fi
official_parameter_file="$(mktemp)"
manifest_parameter_file="$(mktemp)"
official_response_file="$(mktemp)"
manifest_response_file="$(mktemp)"
extract_parameter_signatures "$path" >"$official_parameter_file"
jq -r --arg path "$path" '
.mirror_layer[]
| select(.path == $path)
| .parameter_contract.signatures[]?
' "$MANIFEST_PATH" | sort -u >"$manifest_parameter_file"
if ! cmp -s "$manifest_parameter_file" "$official_parameter_file"; then
while IFS= read -r signature; do
[ -n "$signature" ] || continue
printf '%s\t%s\t%s\tmissing\t%s\n' "$path" "$resource" "$method" "$signature" >>"$parameter_drift_file"
done < <(comm -23 "$manifest_parameter_file" "$official_parameter_file")
while IFS= read -r signature; do
[ -n "$signature" ] || continue
printf '%s\t%s\t%s\tadded\t%s\n' "$path" "$resource" "$method" "$signature" >>"$parameter_drift_file"
done < <(comm -13 "$manifest_parameter_file" "$official_parameter_file")
printf '%s\t%s\n' "$resource" "$method" >>"$affected_methods_file"
fi
extract_response_contract "$path" >"$official_response_file"
jq -c --arg path "$path" '
.mirror_layer[]
| select(.path == $path)
| {
status: (.response_contract.status // null),
field_signatures: (.response_contract.field_signatures // [])
}
' "$MANIFEST_PATH" >"$manifest_response_file"
official_status="$(jq -r '.status // "null"' "$official_response_file")"
manifest_status="$(jq -r '.status // "null"' "$manifest_response_file")"
jq -r '.field_signatures[]?' "$official_response_file" | sort -u >"${official_response_file}.fields"
jq -r '.field_signatures[]?' "$manifest_response_file" | sort -u >"${manifest_response_file}.fields"
if [ "$official_status" != "$manifest_status" ]; then
printf '%s\t%s\t%s\tstatus\tmanifest=%s official=%s\n' "$path" "$resource" "$method" "$manifest_status" "$official_status" >>"$response_drift_file"
printf '%s\t%s\n' "$resource" "$method" >>"$affected_methods_file"
fi
if ! cmp -s "${manifest_response_file}.fields" "${official_response_file}.fields"; then
while IFS= read -r signature; do
[ -n "$signature" ] || continue
printf '%s\t%s\t%s\tmissing\t%s\n' "$path" "$resource" "$method" "$signature" >>"$response_drift_file"
done < <(comm -23 "${manifest_response_file}.fields" "${official_response_file}.fields")
while IFS= read -r signature; do
[ -n "$signature" ] || continue
printf '%s\t%s\t%s\tadded\t%s\n' "$path" "$resource" "$method" "$signature" >>"$response_drift_file"
done < <(comm -13 "${manifest_response_file}.fields" "${official_response_file}.fields")
printf '%s\t%s\n' "$resource" "$method" >>"$affected_methods_file"
fi
rm -f \
"$official_parameter_file" \
"$manifest_parameter_file" \
"$official_response_file" \
"$manifest_response_file" \
"${official_response_file}.fields" \
"${manifest_response_file}.fields"
done <"$manifest_mirror_methods_file"
sort -u "$affected_methods_file" -o "$affected_methods_file"
while IFS=$'\t' read -r resource method; do
[ -n "$resource" ] || continue
[ -n "$method" ] || continue
jq -r --arg resource "$resource" --arg method "$method" '
.convenience_layer[]
| select(.resource == $resource and .base_mirror_method == $method)
| [.resource, .base_mirror_method, (.helpers | join(", "))]
| @tsv
' "$MANIFEST_PATH" >>"$convenience_notes_file"
done <"$affected_methods_file"
sort -u "$convenience_notes_file" -o "$convenience_notes_file"
official_total_paths="$(jq -r '.paths | length' "$openapi_file")"
official_adopted_total_paths="$(wc -l <"$official_adopted_paths_file" | tr -d ' ')"
manifest_summary_official="$(jq -r '.summary.official_total_paths' "$MANIFEST_PATH")"
manifest_summary_adopted="$(jq -r '.summary.adopted_family_total_paths' "$MANIFEST_PATH")"
manifest_summary_implemented="$(jq -r '.summary.implemented_mirror_paths' "$MANIFEST_PATH")"
manifest_summary_open_gaps="$(jq -r '.summary.adopted_family_open_gaps' "$MANIFEST_PATH")"
manifest_mirror_count="$(jq -r '.mirror_layer | length' "$MANIFEST_PATH")"
manifest_gap_count="$(jq -r '.gaps_in_adopted_families | length' "$MANIFEST_PATH")"
if [ "$official_total_paths" != "$manifest_summary_official" ]; then
printf 'summary.official_total_paths\tmanifest=%s\tofficial=%s\n' "$manifest_summary_official" "$official_total_paths" >>"$summary_mismatches_file"
fi
if [ "$official_adopted_total_paths" != "$manifest_summary_adopted" ]; then
printf 'summary.adopted_family_total_paths\tmanifest=%s\tofficial=%s\n' "$manifest_summary_adopted" "$official_adopted_total_paths" >>"$summary_mismatches_file"
fi
if [ "$manifest_mirror_count" != "$manifest_summary_implemented" ]; then
printf 'summary.implemented_mirror_paths\tmanifest=%s\tactual=%s\n' "$manifest_summary_implemented" "$manifest_mirror_count" >>"$summary_mismatches_file"
fi
if [ "$manifest_gap_count" != "$manifest_summary_open_gaps" ]; then
printf 'summary.adopted_family_open_gaps\tmanifest=%s\tactual=%s\n' "$manifest_summary_open_gaps" "$manifest_gap_count" >>"$summary_mismatches_file"
fi
jq -r '.paths | to_entries[] | .value.get.tags[]?' "$openapi_file" | sort -u >"$official_not_adopted_tags_file"
comm -23 "$official_not_adopted_tags_file" "$official_adopted_tags_file" >"${official_not_adopted_tags_file}.filtered"
mv "${official_not_adopted_tags_file}.filtered" "$official_not_adopted_tags_file"
comm -23 "$official_not_adopted_tags_file" "$manifest_gap_tags_file" >"$untracked_official_tags_file"
comm -13 "$official_not_adopted_tags_file" "$manifest_gap_tags_file" >"$stale_manifest_tags_file"
crypto_loc_gap_declared="$(jq -r '[.known_parity_gaps[]? | select(.subject == "crypto::Loc")] | length' "$MANIFEST_PATH")"
if [ "$crypto_loc_gap_declared" -gt 0 ]; then
jq -r "$JQ_LIB
. as \$root
| \$root.components.parameters
| to_entries[]
| select(.value.name == \"loc\")
| select(.key | startswith(\"crypto_\"))
| select((.key | contains(\"perp\")) | not)
| .value.schema[\"\$ref\"]
| select(. != null)
| sub(\"^#/components/schemas/\"; \"\")
| \$root.components.schemas[.].enum[]?
" "$openapi_file" | sort -u >"$official_crypto_loc_file"
jq -r '
.known_parity_gaps[]
| select(.subject == "crypto::Loc")
| .local_values[]
' "$MANIFEST_PATH" | sort -u >"$local_crypto_loc_file"
comm -23 "$official_crypto_loc_file" "$local_crypto_loc_file" >"$missing_crypto_loc_file"
else
: >"$missing_crypto_loc_file"
fi
official_title="$(jq -r '.info.title // .official.title // "unknown"' "$openapi_file")"
official_version="$(jq -r '.info.version // "unknown"' "$openapi_file")"
manifest_generated_at="$(jq -r '.generated_at // "unknown"' "$MANIFEST_PATH")"
missing_contract_count="$(wc -l <"$missing_contract_paths_file" | tr -d ' ')"
stale_mirror_path_count="$(wc -l <"$stale_mirror_paths_file" | tr -d ' ')"
missing_method_count="$(wc -l <"$missing_method_defs_file" | tr -d ' ')"
summary_mismatch_count="$(wc -l <"$summary_mismatches_file" | tr -d ' ')"
parameter_drift_count="$(wc -l <"$parameter_drift_file" | tr -d ' ')"
response_drift_count="$(wc -l <"$response_drift_file" | tr -d ' ')"
untracked_tag_count="$(wc -l <"$untracked_official_tags_file" | tr -d ' ')"
stale_tag_count="$(wc -l <"$stale_manifest_tags_file" | tr -d ' ')"
missing_crypto_loc_count="$(wc -l <"$missing_crypto_loc_file" | tr -d ' ')"
convenience_note_count="$(wc -l <"$convenience_notes_file" | tr -d ' ')"
blocking_count=$((missing_contract_count + stale_mirror_path_count + missing_method_count + summary_mismatch_count + parameter_drift_count + response_drift_count))
echo "alpaca-market-data-sync audit"
echo "Official source: $official_title v$official_version"
echo "Official OpenAPI: $OPENAPI_URL"
echo "Observed at: $OBSERVED_AT"
echo "Local manifest: $MANIFEST_PATH"
echo "Manifest observed_at: $manifest_generated_at"
echo
echo "Summary"
echo "- Official total paths: $official_total_paths"
echo "- Adopted-family official paths: $official_adopted_total_paths"
echo "- Manifest mirror paths: $manifest_mirror_count"
echo "- Manifest adopted-family gaps: $manifest_gap_count"
echo
echo "Findings"
if [ "$blocking_count" -eq 0 ]; then
echo "- No blocking drift detected by the automated audit."
fi
if [ "$manifest_gap_count" -gt 0 ]; then
echo "- The manifest still records $manifest_gap_count adopted-family mirror gap(s) listed below."
fi
if [ "$missing_crypto_loc_count" -gt 0 ]; then
echo "- The manifest still records a crypto::Loc parity gap listed below."
fi
if [ "$blocking_count" -eq 0 ] && [ "$manifest_gap_count" -eq 0 ] && [ "$missing_crypto_loc_count" -eq 0 ] && [ "$untracked_tag_count" -eq 0 ] && [ "$stale_tag_count" -eq 0 ]; then
echo "- No additional warnings or parity notes were detected."
fi
if [ "$missing_contract_count" -gt 0 ]; then
echo
echo "Blocking: official adopted-family paths missing from the local coverage contract"
while IFS= read -r path; do
[ -n "$path" ] || continue
echo "- $path"
done <"$missing_contract_paths_file"
fi
if [ "$stale_mirror_path_count" -gt 0 ]; then
echo
echo "Blocking: local mirror paths no longer exist in the current official OpenAPI"
while IFS= read -r path; do
[ -n "$path" ] || continue
echo "- $path"
done <"$stale_mirror_paths_file"
fi
if [ "$missing_method_count" -gt 0 ]; then
echo
echo "Blocking: mirror methods declared in the manifest but missing from source"
while IFS=$'\t' read -r path resource method client_file; do
[ -n "$path" ] || continue
echo "- $path => ${resource}.${method} expected in $client_file"
done <"$missing_method_defs_file"
fi
if [ "$summary_mismatch_count" -gt 0 ]; then
echo
echo "Blocking: manifest summary fields do not match computed values"
while IFS=$'\t' read -r field manifest_value actual_value; do
[ -n "$field" ] || continue
echo "- $field: $manifest_value, $actual_value"
done <"$summary_mismatches_file"
fi
if [ "$parameter_drift_count" -gt 0 ]; then
echo
echo "Blocking: parameter drift in adopted-family mirror contracts"
while IFS=$'\t' read -r path resource method change_kind signature; do
[ -n "$path" ] || continue
echo "- $path => ${resource}.${method}"
echo " ${change_kind} parameter signature: $signature"
done <"$parameter_drift_file"
fi
if [ "$response_drift_count" -gt 0 ]; then
echo
echo "Blocking: response-field drift in adopted-family mirror contracts"
while IFS=$'\t' read -r path resource method change_kind signature; do
[ -n "$path" ] || continue
if [ "$change_kind" = "status" ]; then
echo "- $path => ${resource}.${method}"
echo " response status drift: $signature"
else
echo "- $path => ${resource}.${method}"
echo " ${change_kind} response field signature: $signature"
fi
done <"$response_drift_file"
fi
if [ "$untracked_tag_count" -gt 0 ]; then
echo
echo "Warning: official resource families not tracked in the manifest"
while IFS= read -r tag; do
[ -n "$tag" ] || continue
echo "- $tag"
done <"$untracked_official_tags_file"
fi
if [ "$stale_tag_count" -gt 0 ]; then
echo
echo "Warning: manifest resource families not found in the current official OpenAPI"
while IFS= read -r tag; do
[ -n "$tag" ] || continue
echo "- $tag"
done <"$stale_manifest_tags_file"
fi
if [ "$manifest_gap_count" -gt 0 ]; then
echo
echo "Open adopted-family mirror gaps"
while IFS= read -r path; do
[ -n "$path" ] || continue
echo "- $path"
done <"$manifest_gap_paths_file"
fi
if [ "$missing_crypto_loc_count" -gt 0 ]; then
echo
echo "Open parity note: official crypto loc values missing from local coverage"
while IFS= read -r value; do
[ -n "$value" ] || continue
echo "- $value"
done <"$missing_crypto_loc_file"
fi
if [ "$convenience_note_count" -gt 0 ]; then
echo
echo "Convenience compatibility notes"
while IFS=$'\t' read -r resource method helpers; do
[ -n "$resource" ] || continue
echo "- ${resource}.${method} helpers require re-validation: $helpers"
done <"$convenience_notes_file"
fi
echo
echo "Recommended changes"
if [ "$missing_contract_count" -gt 0 ] || [ "$summary_mismatch_count" -gt 0 ] || [ "$parameter_drift_count" -gt 0 ] || [ "$response_drift_count" -gt 0 ] || [ "$stale_mirror_path_count" -gt 0 ]; then
echo "- Refresh docs/api-coverage.md and tools/api-coverage/market-data-api.json so the published contract matches the current official OpenAPI."
fi
if [ "$missing_method_count" -gt 0 ]; then
echo "- Implement or remove any manifest mirror entries whose source methods do not exist."
fi
if [ "$manifest_gap_count" -gt 0 ]; then
while IFS= read -r path; do
[ -n "$path" ] || continue
echo "- Implement mirror coverage for adopted-family path: $path."
done <"$manifest_gap_paths_file"
fi
if [ "$missing_crypto_loc_count" -gt 0 ]; then
echo "- Extend crypto::Loc with the missing official values reported above."
fi
if [ "$convenience_note_count" -gt 0 ]; then
echo "- Re-validate the affected *_all and *_stream helpers after mirror-layer parity is restored."
fi
if [ "$blocking_count" -gt 0 ]; then
exit 1
fi