use std::fs::File;
use csv::{QuoteStyle, Writer, WriterBuilder};
use super::provenance::Provenance;
use crate::datasets::sec::error::{Result, SecError};
use crate::datasets::sec::layout::Workdir;
pub const COMPANY_HEADER: &[&str] = &[
"cik",
"name",
"sic",
"sic_description",
"state_of_incorporation",
"fiscal_year_end",
"tickers",
"exchanges",
"entity_type",
"former_names",
];
pub const PERSON_HEADER: &[&str] = &["person_nid", "display_name", "cik"];
pub const SECURITY_HEADER: &[&str] = &["cusip", "name", "title_of_class"];
pub const MANAGER_HEADER: &[&str] = &["manager_cik", "name"];
pub const INSIDER_TRANSACTION_HEADER: &[&str] = &[
"transaction_nid",
"person_nid",
"issuer_cik",
"direction",
"security_title",
"transaction_date",
"transaction_code",
"shares",
"price_per_share",
"total_value",
"direct_indirect",
"is_derivative",
"equity_swap",
"footnote_text",
];
pub const HOLDING_HEADER: &[&str] = &[
"holding_nid",
"person_nid",
"issuer_cik",
"security_title",
"as_of_date",
"shares",
"percent_of_class",
"direct_indirect",
"is_derivative",
];
pub const ROLE_HEADER: &[&str] = &[
"role_nid",
"person_nid",
"issuer_cik",
"role_type", "officer_title",
"since_date",
];
pub const PLANNED_SALE_HEADER: &[&str] = &[
"planned_sale_nid",
"person_nid",
"issuer_cik",
"security_class",
"shares",
"approx_sale_date",
"broker_name",
"aggregate_market_value",
"payment_date",
"securities_acquired_date",
"nature_of_acquisition",
];
pub const INSTITUTIONAL_HOLDING_HEADER: &[&str] = &[
"institutional_holding_nid",
"manager_cik",
"cusip",
"name_of_issuer",
"title_of_class",
"figi",
"value",
"shares",
"shares_type", "put_call", "investment_discretion", "voting_sole",
"voting_shared",
"voting_none",
"other_managers",
"quarter",
];
pub const ACTIVIST_FILING_HEADER: &[&str] = &[
"activist_filing_nid",
"filer_nid", "filer_type", "filer_name",
"issuer_cik",
"security_cusip",
"security_class",
"aggregate_amount",
"percent_of_class",
"sole_voting_power",
"shared_voting_power",
"sole_dispositive_power",
"shared_dispositive_power",
"type_of_reporting_person", "citizenship",
"purpose_text",
"source_of_funds",
"member_of_group",
"is_amendment",
"original_filing_accession",
];
pub const HOLDER_GROUP_HEADER: &[&str] =
&["group_link_nid", "filer_a_nid", "filer_b_nid", "issuer_cik"];
pub const SUBSIDIARY_HEADER: &[&str] = &["subsidiary_nid", "parent_cik", "name", "jurisdiction"];
pub const RELATED_PARTY_TRANSACTION_HEADER: &[&str] = &[
"rpt_nid",
"issuer_cik",
"counterparty_name",
"relationship",
"year",
"amount_usd",
"description",
];
pub const AUDITOR_HEADER: &[&str] = &[
"auditor_record_nid",
"issuer_cik",
"auditor_name",
"auditor_location",
"pcaob_id",
"fiscal_year",
];
pub const CORPORATE_EVENT_HEADER: &[&str] = &[
"event_nid",
"issuer_cik",
"item_code",
"description",
"event_date",
];
pub const OFFICER_CHANGE_HEADER: &[&str] = &[
"officer_change_nid",
"issuer_cik",
"person_name",
"person_nid", "change_type", "position_title",
"effective_date",
"reason_summary",
];
pub const MA_EVENT_HEADER: &[&str] = &[
"ma_event_nid",
"issuer_cik",
"counterparty_name",
"counterparty_cik",
"transaction_type", "effective_date",
"consideration_summary",
"deal_value_usd",
];
pub const VOTE_RESULT_HEADER: &[&str] = &[
"vote_result_nid",
"issuer_cik",
"meeting_date",
"proposal_number",
"proposal_description",
"votes_for",
"votes_against",
"votes_abstain",
"broker_non_votes",
"outcome", ];
pub const AUDITOR_CHANGE_HEADER: &[&str] = &[
"auditor_change_nid",
"issuer_cik",
"prior_auditor",
"new_auditor",
"change_date",
"reason_summary",
];
pub const RESTATEMENT_HEADER: &[&str] = &[
"restatement_nid",
"issuer_cik",
"filing_date",
"period_restated_start",
"period_restated_end",
"items_affected",
"reason_summary",
];
pub const EARNINGS_RELEASE_HEADER: &[&str] = &[
"earnings_release_nid",
"issuer_cik",
"period_end_date",
"fiscal_period",
"revenue",
"net_income",
"eps_basic",
"eps_diluted",
"guidance_revenue_low",
"guidance_revenue_high",
"guidance_eps_low",
"guidance_eps_high",
];
pub const PROPOSAL_HEADER: &[&str] = &[
"proposal_nid",
"issuer_cik",
"meeting_date",
"proposal_number",
"description",
"board_recommendation",
"proposal_type", ];
pub const COMPENSATION_HEADER: &[&str] = &[
"compensation_nid",
"person_name",
"person_nid",
"issuer_cik",
"fiscal_year",
"position_title",
"salary",
"bonus",
"stock_awards",
"option_awards",
"non_equity_incentive",
"pension_change",
"other_compensation",
"total",
];
pub const PAY_VS_PERFORMANCE_HEADER: &[&str] = &[
"pvp_nid",
"issuer_cik",
"fiscal_year",
"ceo_actual_comp",
"ceo_reported_comp",
"avg_neo_actual_comp",
"avg_neo_reported_comp",
"total_shareholder_return",
"peer_tsr",
"net_income",
"company_selected_measure",
];
pub const CEO_PAY_RATIO_HEADER: &[&str] = &[
"ceo_pay_ratio_nid",
"issuer_cik",
"fiscal_year",
"ceo_total_comp",
"median_employee_comp",
"ratio",
];
pub const AUDIT_FEES_HEADER: &[&str] = &[
"audit_fees_nid",
"issuer_cik",
"fiscal_year",
"auditor_name",
"audit_fees",
"audit_related_fees",
"tax_fees",
"other_fees",
];
pub const FUND_VOTE_HEADER: &[&str] = &[
"fund_vote_nid",
"manager_cik",
"series_id",
"issuer_name",
"security_cusip",
"meeting_date",
"proposal_description",
"shares_voted",
"shares_on_loan",
"vote_for",
"vote_against",
"vote_abstain",
"vote_withhold",
"management_recommendation",
"vote_source",
];
pub const OFFERING_HEADER: &[&str] = &[
"offering_nid",
"issuer_cik",
"offering_type", "shares_offered",
"price_per_share",
"gross_proceeds",
"net_proceeds",
"currency",
"is_overallotment_exercised",
];
pub const SELLING_STOCKHOLDER_HEADER: &[&str] = &[
"ss_nid",
"person_nid", "holder_name",
"issuer_cik",
"shares_before",
"shares_offered",
"shares_after",
"pct_before",
"pct_after",
];
pub const UNDERWRITER_HEADER: &[&str] = &[
"underwriter_nid",
"underwriter_name",
"issuer_cik",
"role", "shares_underwritten",
"discount_per_share",
];
pub const USE_OF_PROCEEDS_HEADER: &[&str] = &[
"uop_nid",
"issuer_cik",
"category",
"amount_usd",
"narrative",
];
pub const MERGER_HEADER: &[&str] = &[
"merger_nid",
"target_cik",
"target_name",
"acquirer_cik",
"acquirer_name",
"consideration_type", "cash_per_share",
"exchange_ratio",
"deal_value_usd",
"expected_close_date",
];
pub const METRIC_FACT_HEADER: &[&str] = &[
"metric_fact_nid",
"issuer_cik",
"tag",
"ddate",
"qtrs",
"uom",
"value",
"dimensional_context", ];
pub struct Sinks {
pub company: Writer<File>,
pub person: Writer<File>,
pub security: Writer<File>,
pub manager: Writer<File>,
pub insider_transaction: Writer<File>,
pub holding: Writer<File>,
pub role: Writer<File>,
pub planned_sale: Writer<File>,
pub institutional_holding: Writer<File>,
pub activist_filing: Writer<File>,
pub holder_group: Writer<File>,
pub subsidiary: Writer<File>,
pub related_party_transaction: Writer<File>,
pub auditor: Writer<File>,
pub corporate_event: Writer<File>,
pub officer_change: Writer<File>,
pub ma_event: Writer<File>,
pub vote_result: Writer<File>,
pub auditor_change: Writer<File>,
pub restatement: Writer<File>,
pub earnings_release: Writer<File>,
pub proposal: Writer<File>,
pub compensation: Writer<File>,
pub pay_vs_performance: Writer<File>,
pub ceo_pay_ratio: Writer<File>,
pub audit_fees: Writer<File>,
pub fund_vote: Writer<File>,
pub offering: Writer<File>,
pub selling_stockholder: Writer<File>,
pub underwriter: Writer<File>,
pub use_of_proceeds: Writer<File>,
pub merger: Writer<File>,
pub metric_fact: Writer<File>,
}
impl Sinks {
pub fn open(workdir: &Workdir) -> Result<Self> {
workdir.ensure_dirs(None)?;
let mut sinks = Self {
company: csv_writer(workdir, "company")?,
person: csv_writer(workdir, "person")?,
security: csv_writer(workdir, "security")?,
manager: csv_writer(workdir, "institutional_manager")?,
insider_transaction: csv_writer(workdir, "insider_transaction")?,
holding: csv_writer(workdir, "holding")?,
role: csv_writer(workdir, "role")?,
planned_sale: csv_writer(workdir, "planned_sale")?,
institutional_holding: csv_writer(workdir, "institutional_holding")?,
activist_filing: csv_writer(workdir, "activist_filing")?,
holder_group: csv_writer(workdir, "holder_group")?,
subsidiary: csv_writer(workdir, "subsidiary")?,
related_party_transaction: csv_writer(workdir, "related_party_transaction")?,
auditor: csv_writer(workdir, "auditor")?,
corporate_event: csv_writer(workdir, "corporate_event")?,
officer_change: csv_writer(workdir, "officer_change")?,
ma_event: csv_writer(workdir, "ma_event")?,
vote_result: csv_writer(workdir, "vote_result")?,
auditor_change: csv_writer(workdir, "auditor_change")?,
restatement: csv_writer(workdir, "restatement")?,
earnings_release: csv_writer(workdir, "earnings_release")?,
proposal: csv_writer(workdir, "proposal")?,
compensation: csv_writer(workdir, "compensation")?,
pay_vs_performance: csv_writer(workdir, "pay_vs_performance")?,
ceo_pay_ratio: csv_writer(workdir, "ceo_pay_ratio")?,
audit_fees: csv_writer(workdir, "audit_fees")?,
fund_vote: csv_writer(workdir, "fund_vote")?,
offering: csv_writer(workdir, "offering")?,
selling_stockholder: csv_writer(workdir, "selling_stockholder")?,
underwriter: csv_writer(workdir, "underwriter")?,
use_of_proceeds: csv_writer(workdir, "use_of_proceeds")?,
merger: csv_writer(workdir, "merger")?,
metric_fact: csv_writer(workdir, "metric_fact")?,
};
write_header(&mut sinks.company, COMPANY_HEADER)?;
write_header(&mut sinks.person, PERSON_HEADER)?;
write_header(&mut sinks.security, SECURITY_HEADER)?;
write_header(&mut sinks.manager, MANAGER_HEADER)?;
write_info_header(&mut sinks.insider_transaction, INSIDER_TRANSACTION_HEADER)?;
write_info_header(&mut sinks.holding, HOLDING_HEADER)?;
write_info_header(&mut sinks.role, ROLE_HEADER)?;
write_info_header(&mut sinks.planned_sale, PLANNED_SALE_HEADER)?;
write_info_header(
&mut sinks.institutional_holding,
INSTITUTIONAL_HOLDING_HEADER,
)?;
write_info_header(&mut sinks.activist_filing, ACTIVIST_FILING_HEADER)?;
write_info_header(&mut sinks.holder_group, HOLDER_GROUP_HEADER)?;
write_info_header(&mut sinks.subsidiary, SUBSIDIARY_HEADER)?;
write_info_header(
&mut sinks.related_party_transaction,
RELATED_PARTY_TRANSACTION_HEADER,
)?;
write_info_header(&mut sinks.auditor, AUDITOR_HEADER)?;
write_info_header(&mut sinks.corporate_event, CORPORATE_EVENT_HEADER)?;
write_info_header(&mut sinks.officer_change, OFFICER_CHANGE_HEADER)?;
write_info_header(&mut sinks.ma_event, MA_EVENT_HEADER)?;
write_info_header(&mut sinks.vote_result, VOTE_RESULT_HEADER)?;
write_info_header(&mut sinks.auditor_change, AUDITOR_CHANGE_HEADER)?;
write_info_header(&mut sinks.restatement, RESTATEMENT_HEADER)?;
write_info_header(&mut sinks.earnings_release, EARNINGS_RELEASE_HEADER)?;
write_info_header(&mut sinks.proposal, PROPOSAL_HEADER)?;
write_info_header(&mut sinks.compensation, COMPENSATION_HEADER)?;
write_info_header(&mut sinks.pay_vs_performance, PAY_VS_PERFORMANCE_HEADER)?;
write_info_header(&mut sinks.ceo_pay_ratio, CEO_PAY_RATIO_HEADER)?;
write_info_header(&mut sinks.audit_fees, AUDIT_FEES_HEADER)?;
write_info_header(&mut sinks.fund_vote, FUND_VOTE_HEADER)?;
write_info_header(&mut sinks.offering, OFFERING_HEADER)?;
write_info_header(&mut sinks.selling_stockholder, SELLING_STOCKHOLDER_HEADER)?;
write_info_header(&mut sinks.underwriter, UNDERWRITER_HEADER)?;
write_info_header(&mut sinks.use_of_proceeds, USE_OF_PROCEEDS_HEADER)?;
write_info_header(&mut sinks.merger, MERGER_HEADER)?;
write_info_header(&mut sinks.metric_fact, METRIC_FACT_HEADER)?;
Ok(sinks)
}
pub fn flush_all(&mut self) -> Result<()> {
macro_rules! flush_one {
($w:expr) => {
$w.flush().map_err(SecError::Io)?;
};
}
flush_one!(self.company);
flush_one!(self.person);
flush_one!(self.security);
flush_one!(self.manager);
flush_one!(self.insider_transaction);
flush_one!(self.holding);
flush_one!(self.role);
flush_one!(self.planned_sale);
flush_one!(self.institutional_holding);
flush_one!(self.activist_filing);
flush_one!(self.holder_group);
flush_one!(self.subsidiary);
flush_one!(self.related_party_transaction);
flush_one!(self.auditor);
flush_one!(self.corporate_event);
flush_one!(self.officer_change);
flush_one!(self.ma_event);
flush_one!(self.vote_result);
flush_one!(self.auditor_change);
flush_one!(self.restatement);
flush_one!(self.earnings_release);
flush_one!(self.proposal);
flush_one!(self.compensation);
flush_one!(self.pay_vs_performance);
flush_one!(self.ceo_pay_ratio);
flush_one!(self.audit_fees);
flush_one!(self.fund_vote);
flush_one!(self.offering);
flush_one!(self.selling_stockholder);
flush_one!(self.underwriter);
flush_one!(self.use_of_proceeds);
flush_one!(self.merger);
flush_one!(self.metric_fact);
Ok(())
}
}
const CSV_BUFFER_CAPACITY: usize = 512 * 1024;
fn csv_writer(workdir: &Workdir, name: &str) -> Result<Writer<File>> {
let path = workdir.processed_csv(name);
WriterBuilder::new()
.quote_style(QuoteStyle::Necessary)
.buffer_capacity(CSV_BUFFER_CAPACITY)
.from_path(&path)
.map_err(|e| SecError::Malformed(format!("open {}: {}", path.display(), e)))
}
fn write_header(w: &mut Writer<File>, cols: &[&str]) -> Result<()> {
w.write_record(cols)
.map_err(|e| SecError::Malformed(format!("write header: {}", e)))?;
Ok(())
}
fn write_info_header(w: &mut Writer<File>, type_cols: &[&str]) -> Result<()> {
let mut full: Vec<&str> = type_cols.to_vec();
full.extend_from_slice(Provenance::HEADER);
write_header(w, &full)
}
pub fn write_info_row<S: AsRef<str>>(
w: &mut Writer<File>,
type_cells: &[S],
prov: &Provenance,
) -> Result<()> {
let mut row: Vec<String> = type_cells.iter().map(|c| c.as_ref().to_string()).collect();
for cell in prov.as_cells() {
row.push(cell);
}
w.write_record(&row)
.map_err(|e| SecError::Malformed(format!("write info row: {}", e)))?;
Ok(())
}
pub fn write_identity_row<S: AsRef<str>>(w: &mut Writer<File>, cells: &[S]) -> Result<()> {
let row: Vec<String> = cells.iter().map(|c| c.as_ref().to_string()).collect();
w.write_record(&row)
.map_err(|e| SecError::Malformed(format!("write identity row: {}", e)))?;
Ok(())
}