use std::collections::{BTreeSet, HashMap};
use std::fmt;
use std::str::FromStr;
use serde_json::{Map, Value};
use powerio_core::PioModule;
use crate::diagnostics::{Diagnostic, DiagnosticInfo, Diagnostics, EmitFamily, codes};
use crate::gen_cost::{GenCostPatch, MissingGenCostPolicy};
use crate::network::{
BalancedNetwork, Branch, BranchRatingSet, Bus, BusId, BusType, SourceFormat, Transformer3W,
};
use crate::{Error, Result};
use routing::{Detection, JsonClass, SourceFormat as DetectedFormat, TransmissionFormat};
mod cgmes;
mod decode;
mod egret;
pub(crate) mod goc3;
mod ieee_cdf;
mod matpower;
mod opfdata;
mod pandapower;
mod powermodels;
pub mod powerworld;
mod powsybl_psse;
mod pslf;
pub(crate) mod psse;
mod pypsa;
mod rawx;
pub mod routing;
mod surge;
mod ucte;
mod union_find;
mod xiidm;
mod xml;
pub use powerworld::{PwdDisplay, PwdSubstation};
#[doc(hidden)]
pub use egret::{
egret_declares_time_series as __egret_declares_time_series,
parse_egret_time_series as __parse_egret_time_series,
};
#[doc(hidden)]
pub use opfdata::{OpfDataSolution, parse_opfdata_json as __parse_opfdata_json};
#[doc(hidden)]
pub use pypsa::{
PypsaAxis, PypsaCsvSequence, parse_pypsa_csv_time_series as __parse_pypsa_csv_time_series,
pypsa_axis as __pypsa_axis,
};
pub use cgmes::CgmesVersion;
pub(crate) use egret::write_egret_json;
pub(crate) use pandapower::write_pandapower_json;
pub(crate) use powermodels::write_powermodels_json;
pub(crate) use powerworld::write_powerworld;
pub(crate) use pslf::write_pslf;
pub(crate) use psse::write_psse_rev;
pub(crate) use rawx::write_rawx;
pub(crate) use surge::write_surge_json;
pub(crate) use xiidm::{write_jiidm, write_xiidm};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum TargetFormat {
PowerModelsJson,
EgretJson,
Psse { rev: u32 },
PsseRawx,
PowerWorld,
PandapowerJson,
Matpower,
Pslf,
Goc3Json,
SurgeJson,
DeepMindOpfDataJson,
Xiidm,
Jiidm,
Cgmes,
Ucte,
}
impl TargetFormat {
#[must_use]
pub fn extension(self) -> &'static str {
match self {
TargetFormat::PowerModelsJson
| TargetFormat::EgretJson
| TargetFormat::PandapowerJson
| TargetFormat::Goc3Json
| TargetFormat::SurgeJson
| TargetFormat::DeepMindOpfDataJson => "json",
TargetFormat::Psse { .. } => "raw",
TargetFormat::PsseRawx => "rawx",
TargetFormat::PowerWorld => "aux",
TargetFormat::Matpower => "m",
TargetFormat::Pslf => "epc",
TargetFormat::Xiidm => "xiidm",
TargetFormat::Jiidm => "jiidm",
TargetFormat::Cgmes => "xml",
TargetFormat::Ucte => "uct",
}
}
#[must_use]
pub fn label(self) -> &'static str {
match self {
TargetFormat::PowerModelsJson => "PowerModels JSON",
TargetFormat::EgretJson => "egret JSON",
TargetFormat::Psse { .. } => "PSS/E .raw",
TargetFormat::PsseRawx => "PSS/E RAWX 35",
TargetFormat::PowerWorld => "PowerWorld .aux",
TargetFormat::PandapowerJson => "pandapower JSON",
TargetFormat::Matpower => "MATPOWER .m",
TargetFormat::Pslf => "PSLF .epc",
TargetFormat::Goc3Json => "GO Challenge 3 JSON",
TargetFormat::SurgeJson => "Surge JSON",
TargetFormat::DeepMindOpfDataJson => "DeepMind OPFData JSON",
TargetFormat::Xiidm => "XIIDM 1.17 XML",
TargetFormat::Jiidm => "JIIDM 1.17 JSON",
TargetFormat::Cgmes => "CGMES 3.0 profile set",
TargetFormat::Ucte => "UCTE-DEF .uct",
}
}
#[must_use]
pub fn token(self) -> &'static str {
match self {
TargetFormat::PowerModelsJson => "powermodels-json",
TargetFormat::EgretJson => "egret-json",
TargetFormat::Psse { rev: 34 } => "psse34",
TargetFormat::Psse { rev: 35 } => "psse35",
TargetFormat::Psse { .. } => "psse",
TargetFormat::PsseRawx => "psse-rawx",
TargetFormat::PowerWorld => "powerworld",
TargetFormat::PandapowerJson => "pandapower-json",
TargetFormat::Matpower => "matpower",
TargetFormat::Pslf => "pslf",
TargetFormat::Goc3Json => "goc3-json",
TargetFormat::SurgeJson => "surge-json",
TargetFormat::DeepMindOpfDataJson => "opfdata-json",
TargetFormat::Xiidm => "xiidm",
TargetFormat::Jiidm => "jiidm",
TargetFormat::Cgmes => "cgmes",
TargetFormat::Ucte => "ucte",
}
}
}
impl fmt::Display for TargetFormat {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.token())
}
}
impl FromStr for TargetFormat {
type Err = Error;
fn from_str(name: &str) -> Result<Self> {
parse_target_format(name).ok_or_else(|| Error::UnknownFormat(name.to_string()))
}
}
#[must_use]
pub fn parse_target_format(name: &str) -> Option<TargetFormat> {
if name.eq_ignore_ascii_case("iidm") || name.eq_ignore_ascii_case("rawx") {
return None;
}
Some(match routing::parse_transmission_format(name)? {
TransmissionFormat::Matpower => TargetFormat::Matpower,
TransmissionFormat::PowerModelsJson => TargetFormat::PowerModelsJson,
TransmissionFormat::EgretJson => TargetFormat::EgretJson,
TransmissionFormat::Psse => TargetFormat::Psse { rev: 33 },
TransmissionFormat::Psse34 => TargetFormat::Psse { rev: 34 },
TransmissionFormat::Psse35 => TargetFormat::Psse { rev: 35 },
TransmissionFormat::PsseRawx => TargetFormat::PsseRawx,
TransmissionFormat::PowerWorld => TargetFormat::PowerWorld,
TransmissionFormat::PandapowerJson => TargetFormat::PandapowerJson,
TransmissionFormat::Pslf => TargetFormat::Pslf,
TransmissionFormat::Goc3Json => TargetFormat::Goc3Json,
TransmissionFormat::SurgeJson => TargetFormat::SurgeJson,
TransmissionFormat::DeepMindOpfDataJson => TargetFormat::DeepMindOpfDataJson,
TransmissionFormat::Xiidm => TargetFormat::Xiidm,
TransmissionFormat::Jiidm => TargetFormat::Jiidm,
TransmissionFormat::Cgmes => TargetFormat::Cgmes,
TransmissionFormat::Ucte => TargetFormat::Ucte,
TransmissionFormat::PypsaCsv
| TransmissionFormat::SincalBalanced
| TransmissionFormat::Pwb
| TransmissionFormat::Gridfm
| TransmissionFormat::IeeeCdf => {
return None;
}
})
}
fn parse_source_target_format(name: &str) -> Option<TargetFormat> {
match routing::parse_transmission_format(name) {
Some(TransmissionFormat::Xiidm) => Some(TargetFormat::Xiidm),
Some(TransmissionFormat::PsseRawx) => Some(TargetFormat::PsseRawx),
_ => parse_target_format(name),
}
}
fn display_file_guidance() -> Error {
Error::UnknownFormat(
"a PowerWorld .pwd is display data, not a BalancedNetwork case; \
`powerio::parse` reads it as powerio.GeoLayer"
.into(),
)
}
fn describe_extension(extension: Option<&str>) -> String {
match extension {
Some(ext) => format!("extension `{ext}`"),
None => "no extension".to_owned(),
}
}
fn is_display_format_name(name: &str) -> bool {
matches!(
name.to_ascii_lowercase().replace(['-', '_'], "").as_str(),
"pwd" | "powerworldpwd" | "powerworlddisplay" | "geojson" | "geo" | "geolayer"
)
}
pub fn is_pypsa_csv_name(name: &str) -> bool {
matches!(
name.to_ascii_lowercase().replace(['-', '_'], "").as_str(),
"pypsacsv" | "pypsa"
)
}
fn is_pslf_name(name: &str) -> bool {
matches!(
name.to_ascii_lowercase().replace(['-', '_'], "").as_str(),
"pslf" | "epc" | "pslfepc"
)
}
fn is_ieee_cdf_name(name: &str) -> bool {
routing::parse_transmission_format(name) == Some(TransmissionFormat::IeeeCdf)
}
pub fn parse(
source: powerio_core::Source,
) -> std::result::Result<PioModule<BalancedNetwork>, powerio_core::Error> {
parse_with_json_class(source, None)
}
#[doc(hidden)]
pub fn parse_with_json_class(
source: powerio_core::Source,
json_class: Option<routing::JsonClass>,
) -> std::result::Result<PioModule<BalancedNetwork>, powerio_core::Error> {
if sincal_source_declared(&source) {
return Err(sincal_feature_disabled(&source));
}
let json_class = json_class.or_else(|| {
let buffer = source.primary_buffer().ok()?;
let text = std::str::from_utf8(buffer.content_bytes()).ok()?;
Some(routing::classify_json_text(text))
});
let is_rawx = source
.format()
.and_then(|format| parse_target_format(format.as_str()))
== Some(TargetFormat::PsseRawx)
|| std::path::Path::new(source.name())
.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| extension.eq_ignore_ascii_case("rawx"))
|| matches!(
json_class,
Some(routing::JsonClass::Case(routing::Detection::Known(
routing::SourceFormat::Transmission(TransmissionFormat::PsseRawx)
)))
);
let is_xiidm = source.format().is_some_and(|format| {
routing::parse_transmission_format(format.as_str()) == Some(TransmissionFormat::Xiidm)
}) || std::path::Path::new(source.name())
.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| extension.eq_ignore_ascii_case("xiidm"))
|| (source.format().is_none()
&& source
.primary_buffer()
.is_ok_and(|buffer| xiidm::looks_like_xiidm(buffer.content_bytes())));
let is_jiidm = source.format().is_some_and(|format| {
routing::parse_transmission_format(format.as_str()) == Some(TransmissionFormat::Jiidm)
}) || std::path::Path::new(source.name())
.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| extension.eq_ignore_ascii_case("jiidm"))
|| matches!(
json_class,
Some(routing::JsonClass::Case(routing::Detection::Known(
routing::SourceFormat::Transmission(TransmissionFormat::Jiidm)
)))
);
let is_cgmes = source.format().is_some_and(|format| {
routing::parse_transmission_format(format.as_str()) == Some(TransmissionFormat::Cgmes)
}) || cgmes::looks_like_profile_set(&source);
let mut warnings = Diagnostics::new();
match parse_to_network(&source, &mut warnings, json_class) {
Ok(mut network) => {
network.assign_missing_component_ids();
let source = if is_rawx {
source.with_format(
powerio_core::FormatId::new("psse-rawx")
.expect("the canonical RAWX token is valid"),
)
} else if is_jiidm {
source.with_format(
powerio_core::FormatId::new("jiidm")
.expect("the canonical JIIDM token is valid"),
)
} else if is_xiidm {
source.with_format(
powerio_core::FormatId::new("xiidm")
.expect("the canonical XIIDM token is valid"),
)
} else if is_cgmes {
source.with_format(
powerio_core::FormatId::new("cgmes")
.expect("the canonical CGMES token is valid"),
)
} else if source.format().is_some() {
source
} else {
match powerio_core::FormatId::new(network.source_format().name()) {
Ok(format) => source.with_format(format),
Err(_) => source,
}
};
PioModule::parsed(network, source, warnings.into_records())
}
Err(error) => {
let mut core = powerio_core::Error::new(error.code(), error.to_string());
if let Some(span) = warnings.record_span() {
core = core.with_span(span);
}
Err(core
.with_diagnostics(warnings.into_records())
.with_cause(error)
.with_source(source))
}
}
}
#[allow(clippy::too_many_lines)]
fn parse_to_network(
source: &powerio_core::Source,
warnings: &mut Diagnostics,
json_class: Option<routing::JsonClass>,
) -> Result<BalancedNetwork> {
let from = source.format().map(powerio_core::FormatId::as_str);
let path = std::path::Path::new(source.name());
let stem = if source.name().starts_with('<') {
None
} else if path.extension().is_some() {
path.file_stem().and_then(|stem| stem.to_str())
} else {
Some(source.name())
};
if source.is_directory() {
let marker = powerio_core::ArtifactPath::new("network.csv")
.expect("static name is a valid artifact path");
if from.is_some_and(is_pypsa_csv_name) || (from.is_none() && source.buffer(&marker).is_ok())
{
return pypsa::read_pypsa_csv_source(source, warnings);
}
if from.is_some_and(|format| {
routing::parse_transmission_format(format) == Some(TransmissionFormat::Cgmes)
}) || (from.is_none() && cgmes::looks_like_profile_set(source))
{
return cgmes::parse_source(source, warnings);
}
return Err(Error::UnknownFormat(format!(
"{} is a directory, and the only directory case format is a PyPSA CSV \
folder (one holding a network.csv); pass a case file",
path.display()
)));
}
if from.is_some_and(is_pypsa_csv_name) {
return Err(Error::UnknownFormat(
"a PyPSA CSV case is a directory holding a network.csv; open the folder as the source"
.into(),
));
}
if from.is_some_and(|format| {
routing::parse_transmission_format(format) == Some(TransmissionFormat::Cgmes)
}) || (from.is_none() && cgmes::looks_like_profile_set(source))
{
return cgmes::parse_source(source, warnings);
}
let ext = path
.extension()
.and_then(|e| e.to_str())
.map(str::to_ascii_lowercase);
let looks_like_xiidm = source
.primary_buffer()
.is_ok_and(|buffer| xiidm::looks_like_xiidm(buffer.content_bytes()));
if from.is_some_and(|f| f.eq_ignore_ascii_case("pwb"))
|| (from.is_none() && ext.as_deref() == Some("pwb"))
{
let buffer = primary(source)?;
return powerworld::parse_pwb_collecting(buffer.bytes(), stem, warnings);
}
if from.is_some_and(is_pslf_name) || (from.is_none() && ext.as_deref() == Some("epc")) {
let buffer = primary(source)?;
let network = pslf::parse_pslf_source(source_text(&buffer)?, stem, warnings)?;
reject_empty_case(&network, "PSLF .epc")?;
return Ok(network);
}
if from.is_some_and(is_ieee_cdf_name)
|| (from.is_none()
&& matches!(ext.as_deref(), Some("txt" | "cdf"))
&& source
.primary_buffer()
.is_ok_and(|buffer| ieee_cdf::looks_like_ieee_cdf(buffer.content_bytes())))
{
let buffer = primary(source)?;
let text = source_text(&buffer)?;
let origin = ieee_cdf::TextOrigin::new(
buffer.id().clone(),
(buffer.bytes().len() - buffer.content_bytes().len()) as u64,
);
let network = ieee_cdf::parse_ieee_cdf_source(text, stem, Some(origin), warnings)?;
reject_empty_case(&network, ieee_cdf::FMT)?;
return Ok(network);
}
if from
.and_then(parse_source_target_format)
.is_some_and(|format| format == TargetFormat::DeepMindOpfDataJson)
&& matches!(ext.as_deref(), Some("pt" | "gz"))
{
return Err(Error::UnknownFormat(
"OPFData .pt tensor caches and .tar.gz archives are not case files; extract and parse an example_N.json source file"
.into(),
));
}
if from.is_none() && ext.as_deref() == Some("pwd") {
return Err(display_file_guidance());
}
let fmt_hint = match from {
Some(f) => {
if is_display_format_name(f) {
return Err(display_file_guidance());
}
Some(parse_source_target_format(f).ok_or_else(|| unknown_source_format(f))?)
}
None => {
match ext.as_deref() {
Some("m") => Some(TargetFormat::Matpower),
Some("raw") => Some(TargetFormat::Psse { rev: 33 }),
Some("rawx") => Some(TargetFormat::PsseRawx),
Some("aux") => Some(TargetFormat::PowerWorld),
Some("xiidm") => Some(TargetFormat::Xiidm),
Some("jiidm") => Some(TargetFormat::Jiidm),
Some("xml") if looks_like_xiidm => Some(TargetFormat::Xiidm),
Some("xml" | "zip") => Some(TargetFormat::Cgmes),
Some("uct") => Some(TargetFormat::Ucte),
Some("json") => None,
Some("dss") => return Err(unknown_source_format("dss")),
other => {
let jsonish = source.primary_buffer().is_ok_and(|buffer| {
source_text(&buffer)
.is_ok_and(|text| text.trim_start().starts_with(['{', '[']))
});
if jsonish {
None
} else {
return Err(Error::UnknownFormat(format!(
"cannot infer from source name with {}; \
declare a source format",
describe_extension(other)
)));
}
}
}
}
};
let buffer = primary(source)?;
if fmt_hint == Some(TargetFormat::Xiidm) {
let network = xiidm::parse_xiidm_bytes(buffer.content_bytes(), warnings)?;
reject_empty_case(&network, TargetFormat::Xiidm.label())?;
return Ok(network);
}
let text = source_text(&buffer)?;
warnings.locate_in(
buffer.id().clone(),
(buffer.bytes().len() - buffer.content_bytes().len()) as u64,
);
let fmt = match fmt_hint {
Some(fmt) => fmt,
None => {
json_target_from_class(json_class.unwrap_or_else(|| routing::classify_json_text(text)))?
}
};
if fmt == TargetFormat::Cgmes {
cgmes::check_text_budget(source, text)?;
}
read_source(text, fmt, stem, warnings)
}
fn primary(source: &powerio_core::Source) -> Result<powerio_core::SourceBuffer> {
source.primary_buffer().map_err(|error| Error::FormatRead {
format: "source",
message: error.to_string(),
})
}
fn source_text(buffer: &powerio_core::SourceBuffer) -> Result<&str> {
std::str::from_utf8(buffer.content_bytes()).map_err(|e| Error::FormatRead {
format: "case text",
message: format!("not valid UTF-8: {e}"),
})
}
fn read_source(
text: &str,
fmt: TargetFormat,
name_hint: Option<&str>,
warnings: &mut Diagnostics,
) -> Result<BalancedNetwork> {
let net = match fmt {
TargetFormat::Matpower => matpower::parse_matpower_source(text, name_hint, warnings),
TargetFormat::PowerModelsJson => {
powermodels::parse_powermodels_json_source(text, name_hint, warnings)
}
TargetFormat::Psse { .. } => psse::parse_psse_source(text, name_hint, warnings),
TargetFormat::PsseRawx => rawx::parse_rawx_source(text, name_hint, warnings),
TargetFormat::PowerWorld => {
powerworld::map::parse_powerworld_source(text, name_hint, warnings)
}
TargetFormat::EgretJson => egret::parse_egret_source(text, name_hint),
TargetFormat::PandapowerJson => {
pandapower::parse_pandapower_source(text, name_hint, warnings)
}
TargetFormat::Pslf => pslf::parse_pslf_source(text, name_hint, warnings),
TargetFormat::Goc3Json => {
return Err(Error::UnknownFormat(
"goc3-json defines a GO Challenge 3 calculation; use powerio::parse to obtain AcScucInstance or AcScucSolution"
.into(),
));
}
TargetFormat::SurgeJson => surge::parse_surge_source(text, name_hint, warnings),
TargetFormat::DeepMindOpfDataJson => {
opfdata::parse_opfdata_source(text, name_hint, warnings)
}
TargetFormat::Xiidm => xiidm::parse_xiidm_source(text, warnings),
TargetFormat::Jiidm => xiidm::parse_jiidm_source(text, warnings),
TargetFormat::Cgmes => {
cgmes::parse_text(name_hint.unwrap_or("profile.xml"), text, warnings)
}
TargetFormat::Ucte => ucte::parse_ucte_source(text, name_hint, warnings),
}?;
reject_empty_case(&net, fmt.label())?;
Ok(net)
}
pub(crate) fn geographic_meta(buses: &[Bus]) -> Option<crate::geo::GeoMeta> {
let mut located = buses.iter().filter_map(|bus| bus.location).peekable();
located.peek()?;
let in_bounds = located.all(|location| location.x.abs() <= 180.0 && location.y.abs() <= 90.0);
Some(crate::geo::GeoMeta {
space: if in_bounds {
crate::geo::CoordinateSpace::Geographic { crs: None }
} else {
crate::geo::CoordinateSpace::Unknown
},
kind: None,
})
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
pub(crate) fn id_from_f64(
value: f64,
column: impl std::fmt::Display,
) -> std::result::Result<usize, String> {
if value >= 0.0 && value < i64::MAX as f64 {
Ok(value as usize)
} else {
Err(format!(
"`{column}` value {value:?} is outside the id range 0..2^63"
))
}
}
pub(crate) const SUBSTITUTE_NOMINAL_KV: f64 = 1.0;
pub(crate) fn states_nominal_voltage(bus: &crate::network::Bus) -> bool {
bus.base_kv.is_finite() && bus.base_kv > 0.0
}
pub(crate) fn unstated_nominal_voltage(network: &BalancedNetwork) -> Option<(usize, String)> {
let ids = network
.buses()
.iter()
.filter(|bus| !states_nominal_voltage(bus))
.map(|bus| bus.id.to_string())
.collect::<Vec<_>>();
if ids.is_empty() {
return None;
}
let shown = ids.iter().take(5).map(String::as_str).collect::<Vec<_>>();
let ellipsis = if ids.len() > shown.len() { ", ..." } else { "" };
Some((ids.len(), format!("{}{ellipsis}", shown.join(", "))))
}
pub(crate) fn unbounded_reactive_limits(network: &BalancedNetwork) -> usize {
let generators = network
.generators()
.iter()
.flat_map(|generator| [generator.qmin, generator.qmax]);
let storage = network
.storage()
.iter()
.flat_map(|storage| [storage.qmin, storage.qmax]);
let hvdc = network
.hvdc()
.iter()
.flat_map(|line| [line.qminf, line.qmaxf, line.qmint, line.qmaxt]);
generators
.chain(storage)
.chain(hvdc)
.filter(|value| value.is_infinite())
.count()
}
pub(crate) fn substitute_unbounded_reactive_limits(network: &mut BalancedNetwork) {
fn bound(value: &mut f64) {
if value.is_infinite() {
*value = value.signum() * f64::MAX;
}
}
for generator in network.generators_mut() {
bound(&mut generator.qmin);
bound(&mut generator.qmax);
}
for storage in network.storage_mut() {
bound(&mut storage.qmin);
bound(&mut storage.qmax);
}
for line in network.hvdc_mut() {
bound(&mut line.qminf);
bound(&mut line.qmaxf);
bound(&mut line.qmint);
bound(&mut line.qmaxt);
}
}
pub(crate) fn reject_empty_case(net: &BalancedNetwork, format: &'static str) -> Result<()> {
let detailed = net.detailed_connectivity();
let has_dc_equipment = detailed.as_ref().is_some_and(|detailed| {
!detailed.dc_nodes.is_empty()
|| !detailed.dc_grounds.is_empty()
|| !detailed.dc_lines.is_empty()
|| !detailed.dc_switches.is_empty()
});
let has_empty_xiidm_voltage_level = matches!(
net.source_format(),
SourceFormat::Xiidm | SourceFormat::Jiidm
) && detailed
.as_ref()
.is_some_and(|detailed| !detailed.voltage_levels.is_empty());
if net.buses().is_empty() && !has_dc_equipment && !has_empty_xiidm_voltage_level {
return Err(Error::FormatRead {
format,
message: "case has no buses or DC equipment".into(),
});
}
Ok(())
}
pub const SOURCE_FORMAT_NAMES: &str = "matpower/m, powermodels-json/powermodels/pm, \
egret-json/egret, psse/raw, psse34, psse35, psse-rawx/rawx, powerworld/aux, \
pandapower-json/pandapower/pp, pslf/epc, pypsa-csv/pypsa, pwb, goc3-json/goc3, \
surge-json/surge, opfdata-json/opfdata/gridopt, xiidm/iidm, jiidm, cgmes, ucte/uct, \
ieee-cdf/cdf";
fn unknown_source_format(name: &str) -> Error {
if let Some(dist) = routing::parse_distribution_format(name) {
return Error::UnknownFormat(format!(
"`{}` is a distribution format, and this parser reads only balanced \
transmission formats; parse it through the one module family \
(`powerio::parse` in Rust and `parse` in the language bindings), \
which routes distribution formats",
dist.name()
));
}
Error::UnknownFormat(format!("{name}; accepted names: {SOURCE_FORMAT_NAMES}"))
}
fn json_target_from_class(class: JsonClass) -> Result<TargetFormat> {
match class {
JsonClass::Module => Err(Error::UnknownFormat(
"JSON is PowerIO IR; decode it with `deserialize` rather than the \
grid exchange format parser"
.into(),
)),
JsonClass::Case(Detection::Known(DetectedFormat::Transmission(format))) => {
transmission_json_target(format)
}
JsonClass::Case(Detection::Known(DetectedFormat::Distribution(format))) => {
Err(Error::UnknownFormat(format!(
"JSON looks like distribution `{}`; use the distribution parser or pass an explicit transmission format",
format.name()
)))
}
JsonClass::Case(Detection::Ambiguous) => Err(Error::UnknownFormat(
"ambiguous JSON markers; pass an explicit source format".into(),
)),
JsonClass::Case(Detection::Unknown) => Err(Error::UnknownFormat(
"cannot infer JSON format; pass an explicit source format".into(),
)),
}
}
fn transmission_json_target(format: TransmissionFormat) -> Result<TargetFormat> {
match format {
TransmissionFormat::PowerModelsJson => Ok(TargetFormat::PowerModelsJson),
TransmissionFormat::EgretJson => Ok(TargetFormat::EgretJson),
TransmissionFormat::PandapowerJson => Ok(TargetFormat::PandapowerJson),
TransmissionFormat::Goc3Json => Ok(TargetFormat::Goc3Json),
TransmissionFormat::SurgeJson => Ok(TargetFormat::SurgeJson),
TransmissionFormat::DeepMindOpfDataJson => Ok(TargetFormat::DeepMindOpfDataJson),
TransmissionFormat::PsseRawx => Ok(TargetFormat::PsseRawx),
TransmissionFormat::Jiidm => Ok(TargetFormat::Jiidm),
other => Err(Error::UnknownFormat(format!(
"JSON classifier returned non-JSON transmission format `{}`",
other.name()
))),
}
}
#[derive(Debug, Clone)]
pub(crate) struct TextEmission {
pub(crate) text: String,
pub(crate) diagnostics: Vec<Diagnostic>,
pub(crate) fidelity: powerio_core::Fidelity,
}
impl TextEmission {
pub(crate) fn new(text: String, diagnostics: Diagnostics) -> Self {
Self {
text,
diagnostics: diagnostics.into_records(),
fidelity: powerio_core::Fidelity::Canonical,
}
}
pub(crate) fn faithful(text: String) -> Self {
let mut emission = Self::new(text, Diagnostics::new());
emission.fidelity = powerio_core::Fidelity::ExactSameFormat;
emission
}
#[cfg(test)]
pub(crate) fn render_diagnostics(&self) -> Vec<String> {
crate::diagnostics::render_diagnostics(&self.diagnostics)
}
pub(crate) fn push(&mut self, info: &'static DiagnosticInfo, message: impl Into<String>) {
self.diagnostics.push(Diagnostic::of(info, message));
}
pub(crate) fn prepend(&mut self, read: Vec<Diagnostic>) {
let mut records = read;
records.append(&mut self.diagnostics);
self.diagnostics = records;
}
}
#[derive(Debug, Clone, Default)]
pub struct EmitOptions {
pub missing_gen_cost: MissingGenCostPolicy,
pub gen_cost_patches: Vec<GenCostPatch>,
}
impl EmitOptions {
#[must_use]
pub fn is_default(&self) -> bool {
self.missing_gen_cost.is_preserve() && self.gen_cost_patches.is_empty()
}
}
fn emit_text(
module: &PioModule<BalancedNetwork>,
format: TargetFormat,
) -> std::result::Result<TextEmission, powerio_core::Error> {
if let Some(text) = echo_text(module, format) {
return Ok(TextEmission::faithful(text));
}
let mut conv = emit_value_text(module.value(), format).map_err(core_error)?;
warn_psse_downgrade(module, format, &mut conv);
Ok(conv)
}
pub(crate) fn core_error(error: Error) -> powerio_core::Error {
let message = error.to_string();
powerio_core::Error::new(error.code(), message).with_cause(error)
}
fn echo_text(module: &PioModule<BalancedNetwork>, target: TargetFormat) -> Option<String> {
let source = module.source()?;
let buffer = source.primary_buffer().ok()?;
let source_format = source
.format()
.and_then(|format| parse_target_format(format.as_str()))?;
if !same_target_format(target, source_format) {
return None;
}
let text = std::str::from_utf8(buffer.bytes()).ok()?;
if let TargetFormat::Psse { rev } = target
&& (!matches!(rev, 33..=35)
|| psse::header_rev(text.trim_start_matches('\u{feff}')).ok()? != rev)
{
return None;
}
Some(text.to_owned())
}
fn sincal_source_declared(source: &powerio_core::Source) -> bool {
match source.format() {
Some(format) => {
format.as_str().eq_ignore_ascii_case("sincal")
|| routing::parse_transmission_format(format.as_str())
== Some(TransmissionFormat::SincalBalanced)
}
None => std::path::Path::new(source.name())
.extension()
.is_some_and(|e| e.eq_ignore_ascii_case("sinx")),
}
}
fn sincal_feature_disabled(source: &powerio_core::Source) -> powerio_core::Error {
powerio_core::Error::new(
&codes::REQUEST_SINCAL_FEATURE_DISABLED,
"SINCAL adapters live in powerio-tnd; use powerio with its `sincal` feature or powerio_tnd::formats::sincal",
)
.with_source(source.clone())
}
fn sincal_source_omitted(net: &BalancedNetwork) -> Option<Diagnostic> {
(net.source_format() == SourceFormat::Sincal).then(|| Diagnostic::of(
&codes::EMIT_SINCAL_RETAINED_SOURCE_OMITTED,
"Cross-format emission includes the typed electrical profile; fault, dynamic, protection, diagram, result and other native source-only data are omitted.",
))
}
pub(crate) fn emit_value_text(net: &BalancedNetwork, format: TargetFormat) -> Result<TextEmission> {
let mut conv = match format {
TargetFormat::PowerModelsJson => write_powermodels_json(net),
TargetFormat::EgretJson => write_egret_json(net),
TargetFormat::Psse { rev } => {
if !matches!(rev, 33..=35) {
return Err(Error::Emit {
format: "PSS/E .raw",
message: format!(
"unsupported revision {rev}; emission supports only revisions 33, 34, and 35"
),
});
}
net.check_base_mva()?;
write_psse_rev(net, rev)
}
TargetFormat::PsseRawx => {
net.check_base_mva()?;
write_rawx(net)?
}
TargetFormat::PowerWorld => write_powerworld(net),
TargetFormat::PandapowerJson => write_pandapower_json(net),
TargetFormat::Matpower => matpower::write_matpower_conversion(net),
TargetFormat::Pslf => write_pslf(net),
TargetFormat::SurgeJson => write_surge_json(net),
TargetFormat::Goc3Json => {
return Err(Error::WriteUnsupported {
format: "goc3-json",
});
}
TargetFormat::DeepMindOpfDataJson => {
return Err(Error::WriteUnsupported {
format: "opfdata-json",
});
}
TargetFormat::Xiidm => write_xiidm(net)?,
TargetFormat::Jiidm => write_jiidm(net)?,
TargetFormat::Cgmes => {
return Err(Error::WriteUnsupported { format: "cgmes" });
}
TargetFormat::Ucte => ucte::write_ucte(net)?,
};
conv.diagnostics.extend(sincal_source_omitted(net));
warn_normalized_tap(net, format, &mut conv);
warn_missing_reference(net, format, &mut conv);
warn_dropped_frequency(net, format, &mut conv);
warn_dropped_locations(net, format, &mut conv);
warn_dropped_transformer_charging(net, format, &mut conv);
warn_dropped_exchange_context(net, format, &mut conv);
Ok(conv)
}
pub fn emit(
module: &PioModule<BalancedNetwork>,
format: TargetFormat,
destination: powerio_core::Destination,
) -> std::result::Result<powerio_core::EmitResult, powerio_core::Error> {
emit_with_options(module, format, &EmitOptions::default(), destination)
}
pub fn emit_with_options(
module: &PioModule<BalancedNetwork>,
format: TargetFormat,
options: &EmitOptions,
destination: powerio_core::Destination,
) -> std::result::Result<powerio_core::EmitResult, powerio_core::Error> {
if format == TargetFormat::Cgmes
&& options.is_default()
&& let Some(source) = module.source()
&& source
.format()
.is_some_and(|value| value.as_str() == "cgmes")
{
if source.is_directory() {
let mut artifacts = Vec::new();
for name in source.entry_names()? {
let buffer = source.buffer(&name)?;
artifacts.push(powerio_core::MemoryArtifact::new(
name,
buffer.bytes().to_vec(),
));
}
return destination.__commit_artifacts(
true,
powerio_core::Fidelity::ExactSameFormat,
artifacts,
Vec::new(),
);
}
if source.acquired_buffers().len() == 1 {
let buffer = source.primary_buffer()?;
let file_name = std::path::Path::new(buffer.name())
.file_name()
.and_then(|name| name.to_str())
.and_then(|name| powerio_core::ArtifactPath::new(name).ok())
.unwrap_or_else(|| {
powerio_core::ArtifactPath::new("case.zip")
.expect("static name is a valid artifact path")
});
let artifact = powerio_core::MemoryArtifact::new(file_name, buffer.bytes().to_vec());
return destination.__commit_artifacts(
false,
powerio_core::Fidelity::ExactSameFormat,
vec![artifact],
Vec::new(),
);
}
}
if matches!(format, TargetFormat::Xiidm | TargetFormat::Jiidm)
&& options.is_default()
&& let Some(source) = module.source()
&& source
.format()
.and_then(|value| parse_target_format(value.as_str()))
== Some(format)
&& source.acquired_buffers().len() == 1
{
let buffer = source.primary_buffer()?;
let artifact = powerio_core::MemoryArtifact::new(
powerio_core::ArtifactPath::new(format!("case.{}", format.extension()))
.expect("the format extension names a valid artifact path"),
buffer.bytes().to_vec(),
);
return destination.__commit_artifacts(
false,
powerio_core::Fidelity::ExactSameFormat,
vec![artifact],
Vec::new(),
);
}
if format == TargetFormat::Cgmes {
let (working, mut diagnostics) = if options.is_default() {
(module.value().clone(), Vec::new())
} else {
apply_emit_cost_policy(module.value(), options).map_err(core_error)?
};
let (artifacts, format_diagnostics) = cgmes::artifacts(&working).map_err(core_error)?;
diagnostics.extend(format_diagnostics.into_records());
diagnostics.extend(sincal_source_omitted(&working));
powerio_core::attach_retention_details(
module.diagnostics(),
&mut diagnostics,
"EMIT.SINCAL.RETAINED_SOURCE_OMITTED",
)?;
return destination.__commit_artifacts(
true,
powerio_core::Fidelity::Canonical,
artifacts,
diagnostics,
);
}
let mut conv = emit_text_with_options(module, format, options)?;
powerio_core::attach_retention_details(
module.diagnostics(),
&mut conv.diagnostics,
"EMIT.SINCAL.RETAINED_SOURCE_OMITTED",
)?;
let artifact = powerio_core::MemoryArtifact::new(
powerio_core::ArtifactPath::new("case").expect("static name is a valid artifact path"),
conv.text.into_bytes(),
);
destination.__commit_artifacts(false, conv.fidelity, vec![artifact], conv.diagnostics)
}
#[doc(hidden)]
pub fn __emit_pypsa_csv(
module: &PioModule<BalancedNetwork>,
destination: powerio_core::Destination,
) -> std::result::Result<powerio_core::EmitResult, powerio_core::Error> {
__emit_pypsa_csv_with_options(module, &EmitOptions::default(), destination)
}
#[doc(hidden)]
pub fn __emit_pypsa_csv_with_options(
module: &PioModule<BalancedNetwork>,
options: &EmitOptions,
destination: powerio_core::Destination,
) -> std::result::Result<powerio_core::EmitResult, powerio_core::Error> {
let (working, mut diagnostics) = if options.is_default() {
(None, Vec::new())
} else {
let (network, diagnostics) =
apply_emit_cost_policy(module.value(), options).map_err(core_error)?;
(Some(network), diagnostics)
};
let (artifacts, format_diagnostics) =
pypsa::pypsa_csv_artifacts(working.as_ref().unwrap_or(module.value()));
diagnostics.extend(format_diagnostics);
diagnostics.extend(sincal_source_omitted(module.value()));
powerio_core::attach_retention_details(
module.diagnostics(),
&mut diagnostics,
"EMIT.SINCAL.RETAINED_SOURCE_OMITTED",
)?;
let artifacts = artifacts
.into_iter()
.map(|(name, text)| {
powerio_core::MemoryArtifact::new(
powerio_core::ArtifactPath::new(name).expect("the writer emits fixed valid names"),
text.into_bytes(),
)
})
.collect();
destination.__commit_artifacts(
true,
powerio_core::Fidelity::Canonical,
artifacts,
diagnostics,
)
}
fn emit_text_with_options(
module: &PioModule<BalancedNetwork>,
format: TargetFormat,
options: &EmitOptions,
) -> std::result::Result<TextEmission, powerio_core::Error> {
if options.is_default() {
return emit_text(module, format);
}
let (working, policy_warnings) =
apply_emit_cost_policy(module.value(), options).map_err(core_error)?;
let mut conv = emit_value_text(&working, format).map_err(core_error)?;
conv.prepend(policy_warnings);
Ok(conv)
}
pub(crate) fn apply_emit_cost_policy(
net: &BalancedNetwork,
options: &EmitOptions,
) -> Result<(BalancedNetwork, Vec<Diagnostic>)> {
let mut working = net.clone();
let report =
working.apply_gen_cost_policy(&options.gen_cost_patches, options.missing_gen_cost)?;
let mut policy_warnings = Diagnostics::new();
if report.patched > 0 {
policy_warnings.push(
&codes::TRANSFORM_GEN_COST_POLICY_APPLIED,
format!(
"generator cost patch applied to {} generator(s)",
report.patched
),
);
}
if report.synthesized > 0 {
policy_warnings.push(
&codes::TRANSFORM_GEN_COST_POLICY_APPLIED,
match options.missing_gen_cost {
MissingGenCostPolicy::Fill {
c2,
c1,
c0,
startup,
shutdown,
} => format!(
"generator cost synthesized for {} generator(s): model 2, ncost 3, \
coeffs [{c2}, {c1}, {c0}], startup {startup}, shutdown {shutdown}",
report.synthesized
),
_ => unreachable!("only Fill synthesizes costs"),
},
);
}
Ok((working, policy_warnings.into_records()))
}
pub(crate) fn allocate_circuit_id<K: Ord + Clone>(
preferred: Option<&str>,
key: K,
used: &mut std::collections::BTreeMap<K, std::collections::BTreeSet<String>>,
) -> String {
let taken = used.entry(key).or_default();
if let Some(id) = preferred
&& taken.insert(id.to_owned())
{
return id.to_owned();
}
let mut n = 1u32;
loop {
let candidate = n.to_string();
if taken.insert(candidate.clone()) {
return candidate;
}
n += 1;
}
}
fn warn_psse_downgrade(
module: &PioModule<BalancedNetwork>,
format: TargetFormat,
conv: &mut TextEmission,
) {
let source_text = module
.source()
.and_then(|source| source.primary_buffer().ok())
.and_then(|buffer| String::from_utf8(buffer.content_bytes().to_vec()).ok());
if let (TargetFormat::Psse { rev }, SourceFormat::Psse, Some(src)) = (
format,
module.value().source_format(),
source_text.as_deref(),
) && let Ok(src_rev) = psse::header_rev(src)
&& src_rev > rev
{
conv.push(
&codes::EMIT_PSSE_DOWNGRADED,
format!(
"PSS/E source is revision {src_rev} but the emission target is revision {rev}; \
the older layout drops fields the source carried (emit as psse{src_rev} to keep them)"
),
);
}
}
fn warn_dropped_frequency(net: &BalancedNetwork, format: TargetFormat, conv: &mut TextEmission) {
let carries_frequency = matches!(
format,
TargetFormat::Psse { .. } | TargetFormat::PsseRawx | TargetFormat::PandapowerJson
);
if carries_frequency {
return;
}
if format == TargetFormat::Ucte {
if (net.base_frequency() - 50.0).abs() > 1e-9 {
conv.push(
&format.emit_family().field_dropped,
format!(
"system base frequency {} Hz dropped: UCTE-DEF describes the 50 Hz synchronous area and has no frequency field (reads back as 50 Hz)",
net.base_frequency()
),
);
}
return;
}
if (net.base_frequency() - crate::network::DEFAULT_BASE_FREQUENCY).abs() > 1e-9 {
conv.push(
&format.emit_family().field_dropped,
format!(
"system base frequency {} Hz dropped: {} has no frequency field (reads back as {} Hz)",
net.base_frequency(),
format.label(),
crate::network::DEFAULT_BASE_FREQUENCY
),
);
}
}
fn warn_dropped_locations(net: &BalancedNetwork, format: TargetFormat, conv: &mut TextEmission) {
let carries_locations = matches!(
format,
TargetFormat::PowerWorld | TargetFormat::PandapowerJson
);
if carries_locations {
return;
}
let n = net.buses().iter().filter(|b| b.location.is_some()).count();
let routed = net.branches().iter().filter(|b| b.route.is_some()).count();
if n > 0 || routed > 0 {
conv.push(
&format.emit_family().field_dropped,
format!(
"{n} bus location(s) and {routed} branch route(s) dropped: {} has no \
coordinate field (emit a .geo.json sidecar to keep them)",
format.label()
),
);
}
}
fn warn_dropped_transformer_charging(
net: &BalancedNetwork,
format: TargetFormat,
conv: &mut TextEmission,
) {
if !matches!(format, TargetFormat::Pslf) {
return;
}
let n = net
.branches()
.iter()
.filter(|b| b.is_transformer() && b.calc_total_charging_b() != 0.0)
.count();
if n > 0 {
conv.push(
&codes::EMIT_PSLF.field_dropped,
format!(
"{n} transformer(s) carry line charging that the PSLF .epc transformer \
record cannot represent; the charging was dropped"
),
);
}
}
fn warn_dropped_exchange_context(
net: &BalancedNetwork,
format: TargetFormat,
conv: &mut TextEmission,
) {
if !matches!(
format,
TargetFormat::Matpower
| TargetFormat::PowerModelsJson
| TargetFormat::EgretJson
| TargetFormat::PowerWorld
| TargetFormat::PandapowerJson
| TargetFormat::Pslf
| TargetFormat::SurgeJson
| TargetFormat::Ucte
) {
return;
}
if let Some(message) = exchange_context_loss(net, format.label()) {
conv.push(&format.emit_family().field_dropped, message);
}
}
#[allow(clippy::too_many_lines)] pub(super) fn exchange_context_loss(net: &BalancedNetwork, target: &str) -> Option<String> {
let mut dropped = Vec::new();
let metadata = net.case_metadata();
let mut metadata_fields = Vec::new();
if metadata.case_date.is_some() {
metadata_fields.push("case_date");
}
if metadata.forecast_distance.is_some() {
metadata_fields.push("forecast_distance");
}
if metadata.source_model_format.is_some() {
metadata_fields.push("source_model_format");
}
if metadata.minimum_validation_level.is_some() {
metadata_fields.push("minimum_validation_level");
}
if !metadata_fields.is_empty() {
dropped.push(format!(
"case metadata fields `{}`",
metadata_fields.join("`, `")
));
}
if let Some(detailed) = net.detailed_connectivity().as_deref() {
let records = [
detailed.omitted_fields.len(),
detailed.component_metadata.len(),
detailed.subnetworks.len(),
detailed.substations.len(),
detailed.voltage_levels.len(),
detailed.bus_breaker_buses.len(),
detailed.calculated_buses.len(),
detailed.connectivity_nodes.len(),
detailed.busbar_sections.len(),
detailed.junctions.len(),
detailed.terminals.len(),
detailed.switches.len(),
detailed.internal_connections.len(),
detailed.operational_limit_groups.len(),
detailed.tap_changers.len(),
detailed.equipment_reactive_limits.len(),
detailed.boundary_lines.len(),
detailed.tie_lines.len(),
detailed.dc_converter_units.len(),
detailed.dc_topological_nodes.len(),
detailed.dc_nodes.len(),
detailed.dc_grounds.len(),
detailed.dc_busbars.len(),
detailed.dc_lines.len(),
detailed.dc_series_devices.len(),
detailed.dc_switches.len(),
detailed.voltage_source_converters.len(),
detailed.line_commutated_converters.len(),
]
.into_iter()
.sum::<usize>();
if records > 0 {
dropped.push(format!(
"the `detailed_connectivity` hierarchy and topology ({records} record(s))"
));
}
}
let source_uids = net
.buses()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.loads()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.shunts()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.static_var_compensators()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.branches()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.switches()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.generators()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.storage()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.hvdc()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.transformers_3w()
.iter()
.filter(|value| value.uid.is_some() && !net.uid_is_generated(value.uid.as_deref()))
.count()
+ net
.areas()
.iter()
.filter(|value| value.uid.is_some())
.count();
if source_uids > 0 {
dropped.push(format!(
"{source_uids} source-assigned stable component identity value(s)"
));
}
if net.geo().is_some() {
dropped.push("geographic coordinate reference metadata".to_owned());
}
if net.solver().is_some() {
dropped.push("solver and solution-control metadata".to_owned());
}
if dropped.is_empty() {
return None;
}
Some(format!(
"{target} has no complete source-neutral grid exchange context; dropped {}",
dropped.join(", ")
))
}
pub(super) fn branch_rating_set_drop_warning(
target: &str,
branch_index: usize,
branch: &Branch,
rating: &BranchRatingSet,
) -> String {
format!(
"branch {} ({} to {}) rating set {}={} MVA dropped: {} has no field for branch rating sets beyond rate_a, rate_b, and rate_c",
branch_index + 1,
branch.from,
branch.to,
rating.name,
rating.rate_mva,
target
)
}
pub(super) fn warn_dropped_extras(
family: &'static EmitFamily,
target: &str,
net: &BalancedNetwork,
consumed: impl Fn(&str) -> bool,
warnings: &mut Diagnostics,
) {
let mut dropped = 0usize;
let mut keys: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
for extras in net
.buses()
.iter()
.map(|e| &e.extras)
.chain(net.branches().iter().map(|e| &e.extras))
.chain(net.loads().iter().map(|e| &e.extras))
.chain(net.shunts().iter().map(|e| &e.extras))
.chain(net.switches().iter().map(|e| &e.extras))
.chain(net.storage().iter().map(|e| &e.extras))
.chain(net.hvdc().iter().map(|e| &e.extras))
.chain(net.transformers_3w().iter().map(|e| &e.extras))
{
let mut carries = false;
for key in extras.keys().filter(|key| !consumed(key)) {
carries = true;
keys.insert(key.as_str());
}
if carries {
dropped += 1;
}
}
if dropped > 0 {
let named = keys
.iter()
.take(EXTRAS_KEYS_NAMED)
.copied()
.collect::<Vec<_>>()
.join("`, `");
let remainder = keys.len().saturating_sub(EXTRAS_KEYS_NAMED);
let remainder = if remainder > 0 {
format!(" and {remainder} more")
} else {
String::new()
};
warnings.push(
&family.extras_dropped,
format!(
"{dropped} element(s) state field(s) `{named}`{remainder} that no {target} record \
holds; dropped"
),
);
}
}
const EXTRAS_KEYS_NAMED: usize = 6;
pub(super) fn warn_dropped_areas(
family: &'static EmitFamily,
target: &str,
writes_bus_area: bool,
net: &BalancedNetwork,
warnings: &mut Diagnostics,
) {
if !writes_bus_area {
if !net.areas().is_empty() {
let count = net.areas().len();
let noun = if count == 1 { "record" } else { "records" };
warnings.push(
&family.areas_dropped,
format!(
"{count} area {noun} dropped: {target} writes neither an area table nor a bus area number"
),
);
}
return;
}
let mut fields = std::collections::BTreeSet::new();
let mut stated = 0usize;
for area in net.areas() {
let mut states_any = false;
for field in stated_area_fields(area) {
fields.insert(field);
states_any = true;
}
stated += usize::from(states_any);
}
if fields.is_empty() {
return;
}
let total = net.areas().len();
let subject = if total == 1 {
"the area record states".to_owned()
} else {
format!("{stated} of {total} area records state")
};
warnings.push(
&family.areas_dropped,
format!(
"{subject} {}: {target} bus rows carry the area number, but {target} has no record \
for the area's attributes",
fields.into_iter().collect::<Vec<_>>().join(", ")
),
);
}
fn stated_area_fields(area: &crate::Area) -> impl Iterator<Item = &'static str> {
[
(area.slack_bus.is_some(), "swing bus"),
(area.net_interchange != 0.0, "scheduled net interchange"),
(area.tolerance != 0.0, "interchange tolerance"),
(area.name.is_some(), "name"),
(area.uid.is_some(), "source identity"),
(area.area_type.is_some(), "classification"),
]
.into_iter()
.filter_map(|(stated, field)| stated.then_some(field))
}
pub(super) fn warn_extra_branch_rating_sets(
family: &'static EmitFamily,
target: &str,
net: &BalancedNetwork,
warnings: &mut Diagnostics,
) {
for (branch_index, branch) in net.branches().iter().enumerate() {
for rating in &branch.rating_sets {
warnings.push(
&family.rating_set_dropped,
branch_rating_set_drop_warning(target, branch_index, branch, rating),
);
}
}
}
pub(super) fn warn_winding_rating_sets(
family: &'static EmitFamily,
target: &str,
net: &BalancedNetwork,
warnings: &mut Diagnostics,
) {
for transformer in net.transformers_3w() {
let [i, j, k] = transformer.windings.each_ref().map(|winding| winding.bus);
for index in 0..3 {
for rating in transformer.winding_rating_sets(index) {
warnings.push(
&family.rating_set_dropped,
format!(
"three winding transformer {i}-{j}-{k} winding {} rating set {}={} MVA dropped: {target} has no field for winding rating sets beyond rate_a, rate_b, and rate_c",
index + 1,
rating.name,
rating.rate_mva
),
);
}
}
}
}
pub(super) fn warn_winding_status_collapsed(
family: &'static EmitFamily,
target: &str,
net: &BalancedNetwork,
warnings: &mut Diagnostics,
) {
let windings = net
.transformers_3w()
.iter()
.filter(|transformer| transformer.in_service)
.flat_map(Transformer3W::winding_in_service)
.filter(|in_service| !in_service)
.count();
if windings > 0 {
warnings.push(
&family.value_collapsed,
format!(
"{windings} out of service three winding transformer winding(s) written in service: the {target} writer states one status per three winding transformer"
),
);
}
}
pub fn parse_format_id(
token: &str,
) -> std::result::Result<powerio_core::FormatId, powerio_core::Error> {
powerio_core::FormatId::new(token.to_ascii_lowercase().replace('_', "-"))
}
fn warn_missing_reference(net: &BalancedNetwork, format: TargetFormat, conv: &mut TextEmission) {
let needs_ref = matches!(
format,
TargetFormat::Matpower
| TargetFormat::Psse { .. }
| TargetFormat::PsseRawx
| TargetFormat::PowerModelsJson
| TargetFormat::PandapowerJson
| TargetFormat::Pslf
| TargetFormat::SurgeJson
);
if needs_ref && let Some(message) = missing_reference_warning(net) {
conv.push(&format.emit_family().reference_missing, message);
}
}
pub(super) fn missing_reference_warning(net: &BalancedNetwork) -> Option<String> {
(!net.buses().iter().any(|b| b.kind == BusType::Ref)).then(|| {
"no reference (slack) bus in the source network; power flow tools \
reject such cases; to_normalized synthesizes a slack at the \
largest pmax in service generator bus"
.to_string()
})
}
#[allow(clippy::float_cmp)]
fn warn_normalized_tap(net: &BalancedNetwork, format: TargetFormat, conv: &mut TextEmission) {
if matches!(format, TargetFormat::Matpower) {
return;
}
if let Some(message) = normalized_tap_warning(net) {
conv.push(&format.emit_family().element_relabeled, message);
}
}
#[allow(clippy::float_cmp)]
pub(super) fn normalized_tap_warning(net: &BalancedNetwork) -> Option<String> {
if !net.is_normalized() {
return None;
}
let ambiguous = net
.branches()
.iter()
.filter(|b| b.tap == 1.0 && b.shift == 0.0)
.count();
(ambiguous > 0).then(|| {
format!(
"normalized network: {ambiguous} branch(es) have unit tap and no phase \
shift, so the line/transformer label is not preserved (the power flow \
is identical)"
)
})
}
pub(super) fn nonzero_differs(value: f64, reference: f64) -> bool {
value.abs() > f64::EPSILON && (value - reference).abs() > f64::EPSILON
}
pub(crate) fn set_bus_kind(
buses: &mut [Bus],
bus_pos: &HashMap<BusId, usize>,
bus: BusId,
kind: BusType,
) {
if let Some(&idx) = bus_pos.get(&bus)
&& buses[idx].kind != BusType::Isolated
{
buses[idx].kind = kind;
}
}
pub(crate) fn bus_kv(buses: &[Bus], bus_pos: &HashMap<BusId, usize>, bus: BusId) -> f64 {
bus_pos
.get(&bus)
.and_then(|&i| buses.get(i))
.map_or(0.0, |b| b.base_kv)
}
pub(crate) fn sanitize_quoted<'a>(
value: &'a str,
forbidden: &[char],
replacement: char,
) -> std::borrow::Cow<'a, str> {
let breaks = |c: char| c == '\n' || c == '\r' || forbidden.contains(&c);
if value.contains(breaks) {
value
.chars()
.map(|c| if breaks(c) { replacement } else { c })
.collect::<String>()
.into()
} else {
std::borrow::Cow::Borrowed(value)
}
}
pub(crate) fn zbase(v_kv: f64, base_mva: f64) -> f64 {
if v_kv > 0.0 && base_mva > 0.0 {
v_kv * v_kv / base_mva
} else {
1.0
}
}
fn same_target_format(requested: TargetFormat, source: TargetFormat) -> bool {
requested == source
|| matches!(
(requested, source),
(TargetFormat::Psse { .. }, TargetFormat::Psse { .. })
)
}
pub(crate) fn jnum(x: f64) -> Value {
serde_json::Number::from_f64(x).map_or(Value::Null, Value::Number)
}
pub(crate) fn finish(
family: &'static EmitFamily,
root: Map<String, Value>,
mut warnings: Diagnostics,
) -> TextEmission {
let value = Value::Object(root);
let mut nulls = BTreeSet::new();
collect_null_keys(&value, &mut nulls);
if !nulls.is_empty() {
warnings.push(
&family.not_a_number,
format!(
"non-finite numeric values written as JSON null in field(s): {}",
nulls.into_iter().collect::<Vec<_>>().join(", ")
),
);
}
let text = serde_json::to_string_pretty(&value).expect("a serde_json::Value always serializes");
TextEmission::new(text, warnings)
}
fn collect_null_keys(value: &Value, out: &mut BTreeSet<String>) {
match value {
Value::Object(map) => {
for (key, val) in map {
if val.is_null() {
out.insert(key.clone());
} else {
collect_null_keys(val, out);
}
}
}
Value::Array(items) => items.iter().for_each(|v| collect_null_keys(v, out)),
_ => {}
}
}
#[cfg(test)]
pub(crate) mod test_parse {
use super::*;
#[derive(Debug)]
pub(crate) struct TestParsed {
pub network: BalancedNetwork,
pub diagnostics: Vec<Diagnostic>,
}
impl TestParsed {
pub(crate) fn render_diagnostics(&self) -> Vec<String> {
crate::diagnostics::render_diagnostics(&self.diagnostics)
}
}
fn declared(
source: powerio_core::Source,
from: Option<&str>,
) -> std::result::Result<powerio_core::Source, powerio_core::Error> {
match from {
None => Ok(source),
Some(token) => Ok(source.with_format(powerio_core::FormatId::new(
token.to_ascii_lowercase().replace('_', "-"),
)?)),
}
}
pub(crate) fn parse_file(
path: impl AsRef<std::path::Path>,
from: Option<&str>,
) -> std::result::Result<TestParsed, powerio_core::Error> {
let source = declared(powerio_core::Source::open(path.as_ref())?, from)?;
parse(source).map(|module| TestParsed {
diagnostics: module.diagnostics().to_vec(),
network: module.into_value(),
})
}
pub(crate) fn parse_str(
text: &str,
from: &str,
) -> std::result::Result<TestParsed, powerio_core::Error> {
let source = declared(
powerio_core::Source::from_memory("<memory>", text.as_bytes().to_vec())?,
Some(from),
)?;
parse(source).map(|module| TestParsed {
diagnostics: module.diagnostics().to_vec(),
network: module.into_value(),
})
}
}
#[cfg(test)]
mod tests {
use super::test_parse::{parse_file, parse_str};
use super::*;
use crate::network::SourceFormat;
#[test]
fn sanitize_quoted_always_replaces_line_terminators() {
for forbidden in [&[][..], &['\''][..], &['"'][..]] {
let out = sanitize_quoted("A\n42, 'X'\r\nB", forbidden, ' ');
assert!(
!out.contains('\n') && !out.contains('\r'),
"terminator survived with forbidden={forbidden:?}: {out:?}"
);
}
assert!(matches!(
sanitize_quoted("clean name", &['\''], ' '),
std::borrow::Cow::Borrowed(_)
));
}
#[test]
fn dss_extension_error_names_the_distribution_surface() {
let path = std::env::temp_dir().join(format!(
"powerio-dss-surface-{}-feeder.dss",
std::process::id()
));
std::fs::write(&path, "New Circuit.feeder\n").unwrap();
let err = parse_file(&path, None).unwrap_err();
let _ = std::fs::remove_file(&path);
assert!(err.to_string().contains("distribution"), "got: {err}");
}
#[test]
fn io_error_names_the_path() {
let path =
std::env::temp_dir().join(format!("powerio-no-such-case-{}.m", std::process::id()));
let err = parse_file(&path, None).unwrap_err();
assert_eq!(err.category(), powerio_core::ErrorCategory::Io);
let msg = err.to_string();
assert!(
msg.contains(&path.display().to_string()),
"the io failure must name the path: {msg}"
);
}
#[test]
fn a_directory_is_refused_as_a_directory() {
let dir = std::env::temp_dir().join(format!("pglib-opf-23.07-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let err = parse_file(&dir, None).unwrap_err();
std::fs::remove_dir_all(&dir).unwrap();
let msg = err.to_string();
assert!(msg.contains("is a directory"), "got: {msg}");
assert!(msg.contains(&dir.display().to_string()), "got: {msg}");
assert!(msg.contains("PyPSA CSV folder"), "got: {msg}");
}
#[test]
fn unknown_format_error_lists_the_accepted_names() {
let err = parse_str("anything", "not-a-format").unwrap_err();
let msg = err.to_string();
assert!(msg.contains("not-a-format"), "got: {msg}");
assert!(msg.contains("accepted names:"), "got: {msg}");
assert!(msg.contains(SOURCE_FORMAT_NAMES), "got: {msg}");
}
#[test]
fn the_accepted_name_list_matches_the_matcher() {
use routing::TransmissionFormat as TF;
let mut canonical = Vec::new();
for clause in SOURCE_FORMAT_NAMES.split(", ") {
for (i, alias) in clause.split('/').enumerate() {
let resolved = routing::parse_transmission_format(alias);
assert!(
resolved.is_some(),
"listed alias `{alias}` does not resolve"
);
if i == 0 {
canonical.push(resolved.unwrap());
}
}
}
for format in [
TF::Matpower,
TF::PowerModelsJson,
TF::EgretJson,
TF::Psse,
TF::Psse34,
TF::Psse35,
TF::PowerWorld,
TF::PandapowerJson,
TF::PypsaCsv,
TF::Pslf,
TF::Pwb,
TF::Goc3Json,
TF::SurgeJson,
TF::DeepMindOpfDataJson,
TF::Ucte,
TF::Xiidm,
TF::Jiidm,
TF::Cgmes,
TF::IeeeCdf,
] {
assert!(
canonical.contains(&format),
"{} is missing from SOURCE_FORMAT_NAMES",
format.name()
);
}
}
#[test]
fn a_case_with_generators_and_no_cost_data_warns() {
let costless = "\
function mpc = nocost
mpc.version = '2';
mpc.baseMVA = 100;
mpc.bus = [
\t1\t3\t0\t0\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;
\t2\t1\t50\t10\t0\t0\t1\t1\t0\t230\t1\t1.1\t0.9;
];
mpc.gen = [
\t1\t60\t0\t100\t-100\t1\t100\t1\t100\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0;
];
mpc.branch = [
\t1\t2\t0.01\t0.1\t0\t0\t0\t0\t0\t0\t1\t-360\t360;
];
";
let parsed = parse_str(costless, "matpower").unwrap();
assert!(
parsed.render_diagnostics().is_empty(),
"{:?}",
parsed.render_diagnostics()
);
let normalized = parsed
.network
.to_normalized_with_options(&crate::NormalizeOptions::default())
.unwrap();
let absent: Vec<_> = normalized
.diagnostics
.iter()
.filter(|d| d.code() == "CANONICALIZE.NORMALIZE.GEN_COST_ABSENT")
.collect();
assert_eq!(absent.len(), 1, "{:?}", normalized.warnings);
assert!(absent[0].message().contains("no cost data"), "{absent:?}");
assert!(absent[0].message().contains("1 in-service"), "{absent:?}");
let costed = format!("{costless}mpc.gencost = [\n\t2\t0\t0\t3\t0.01\t40\t0;\n];\n");
let parsed = parse_str(&costed, "matpower").unwrap();
let normalized = parsed
.network
.to_normalized_with_options(&crate::NormalizeOptions::default())
.unwrap();
assert!(
normalized
.diagnostics
.iter()
.all(|d| d.code() != "CANONICALIZE.NORMALIZE.GEN_COST_ABSENT"),
"{:?}",
normalized.warnings
);
}
#[test]
fn distribution_from_token_error_names_the_distribution_surface() {
for token in ["dss", "pmd", "bmopf"] {
let err = parse_str("anything", token).unwrap_err();
assert!(
err.to_string().contains("one module family"),
"{token}: {err}"
);
}
let err = parse_str("anything", "nonesuch").unwrap_err();
assert!(err.to_string().contains("nonesuch"));
}
#[test]
fn byte_order_mark_is_retained_and_echoed() {
let case = "\u{feff}function mpc = t\n\
mpc.version = '2';\n\
mpc.baseMVA = 100;\n\
mpc.bus = [1 3 0 0 0 0 1 1.0 0 345 1 1.1 0.9;];\n\
mpc.gen = [];\n\
mpc.branch = [];\n";
let source = powerio_core::Source::from_memory("case.m", case.as_bytes().to_vec()).unwrap();
let module = parse(source.with_format(parse_format_id("matpower").unwrap())).unwrap();
assert_eq!(module.value().buses().len(), 1);
assert!(
module.diagnostics().is_empty(),
"{:?}",
module.diagnostics()
);
let echo = emit_text(&module, TargetFormat::Matpower).unwrap();
assert_eq!(echo.text, case, "the echo reproduces the mark exactly");
}
#[test]
fn canonical_format_bypasses_same_format_matpower_echo() {
let case = "function mpc = t\n\
% a comment the canonical writer does not keep\n\
mpc.version = '2';\n\
mpc.baseMVA = 100;\n\
mpc.bus = [1 3 0 0 0 0 1 1.0 0 345 1 1.1 0.9;];\n\
mpc.gen = [];\n\
mpc.branch = [];\n";
let source = powerio_core::Source::from_memory("case.m", case.as_bytes().to_vec()).unwrap();
let module =
parse(source.with_format(powerio_core::FormatId::new("matpower").unwrap())).unwrap();
assert_eq!(
emit_text(&module, TargetFormat::Matpower).unwrap().text,
case
);
let net = module.into_value();
let canonical = emit_value_text(&net, TargetFormat::Matpower).unwrap();
assert_ne!(canonical.text, case);
let reparsed = parse_str(&canonical.text, "matpower").unwrap();
assert_eq!(reparsed.network.buses().len(), 1);
}
#[test]
fn bus_branch_formats_report_dropped_exchange_context_once() {
let case = "function mpc = t\n\
mpc.version = '2';\n\
mpc.baseMVA = 100;\n\
mpc.bus = [1 3 0 0 0 0 1 1.0 0 345 1 1.1 0.9;];\n\
mpc.gen = [];\n\
mpc.branch = [];\n";
let mut net = parse_str(case, "matpower").unwrap().network;
*net.detailed_connectivity_mut() =
Some(std::sync::Arc::new(crate::DetailedConnectivity::default()));
assert!(exchange_context_loss(&net, "PowerModels JSON").is_none());
net.case_metadata_mut().case_date = Some("2025-01-02T03:04:05Z".into());
net.buses_mut()[0].uid = Some("source-bus".into());
let mut detailed = crate::DetailedConnectivity::default();
detailed.substations.push(crate::Substation {
component: powerio_core::ComponentId::new("substation", "source-substation").unwrap(),
country: None,
operator: None,
geographical_tags: Vec::new(),
});
*net.detailed_connectivity_mut() = Some(std::sync::Arc::new(detailed));
let emitted = emit_value_text(&net, TargetFormat::PowerModelsJson).unwrap();
let context = emitted
.diagnostics
.iter()
.filter(|diagnostic| {
diagnostic.code() == "EMIT.POWERMODELS.FIELD_DROPPED"
&& diagnostic.message().contains("grid exchange context")
})
.collect::<Vec<_>>();
assert_eq!(context.len(), 1, "{:?}", emitted.render_diagnostics());
let message = context[0].message();
assert!(message.contains("case_date"), "{message}");
assert!(message.contains("detailed_connectivity"), "{message}");
assert!(message.contains("1 source-assigned"), "{message}");
}
#[test]
fn source_format_strings_round_trip_to_a_target() {
for (sf, want) in [
(SourceFormat::Matpower, TargetFormat::Matpower),
(SourceFormat::PowerModelsJson, TargetFormat::PowerModelsJson),
(SourceFormat::EgretJson, TargetFormat::EgretJson),
(SourceFormat::Psse, TargetFormat::Psse { rev: 33 }),
(SourceFormat::PowerWorld, TargetFormat::PowerWorld),
(SourceFormat::PandapowerJson, TargetFormat::PandapowerJson),
(SourceFormat::Pslf, TargetFormat::Pslf),
(SourceFormat::Goc3Json, TargetFormat::Goc3Json),
(SourceFormat::SurgeJson, TargetFormat::SurgeJson),
(SourceFormat::Ucte, TargetFormat::Ucte),
(
SourceFormat::DeepMindOpfDataJson,
TargetFormat::DeepMindOpfDataJson,
),
] {
let token = sf.name();
assert_eq!(
parse_target_format(token),
Some(want),
"source_format {token:?} did not round-trip"
);
}
for sf in [
SourceFormat::InMemory,
SourceFormat::Normalized,
SourceFormat::Gridfm,
SourceFormat::PypsaCsv,
SourceFormat::PowerWorldBinary,
SourceFormat::IeeeCdf,
] {
assert_eq!(parse_target_format(sf.name()), None);
}
}
#[test]
fn unsupported_indexed_assignment_warning_preserves_public_literals_and_echo() {
let case = "function mpc = t\n\
mpc.version = '2';\n\
mpc.baseMVA = 100;\n\
mpc.bus = [1 3 12 4 0 0 1 1.0 0 345 1 1.1 0.9;];\n\
mpc.gen = [];\n\
mpc.branch = [];\n\
mpc.bus(:, [3 4]) = mpc.bus(:, [3 4]) / unknown_scale;\n";
let source = powerio_core::Source::from_memory("case.m", case.as_bytes().to_vec()).unwrap();
let module = parse(source.with_format(powerio_core::FormatId::new("matpower").unwrap()))
.expect("literal case parses with a diagnostic");
assert!(
module.diagnostics().iter().any(|d| {
d.code() == "READ.MATPOWER.INDEXED_ASSIGNMENT"
&& d.severity() == powerio_core::DiagnosticSeverity::Warning
}),
"the warning must be present on the returned public module: {:?}",
module.diagnostics()
);
let demand: f64 = module.value().loads().iter().map(|load| load.p).sum();
assert!(
(demand - 12.0).abs() < 1e-12,
"unsupported conversion must not mutate literal values: {demand}"
);
assert_eq!(
emit_text(&module, TargetFormat::Matpower).unwrap().text,
case,
"same-format emission must preserve the exact source bytes"
);
}
}