pub mod altw;
pub mod apply_changes;
pub mod cat;
#[cfg(feature = "commands")]
pub mod check;
pub mod degrade;
pub mod diff;
#[cfg(feature = "commands")]
pub mod export;
#[cfg(feature = "commands")]
pub mod extract;
pub mod getid;
pub mod getparents;
pub mod inspect;
pub mod merge_changes;
pub mod renumber;
pub mod repack;
pub mod sort;
#[cfg(feature = "commands")]
pub(crate) mod spatial;
pub mod tags_count;
pub mod tags_filter;
pub mod time_filter;
use std::path::Path;
use crate::blob::BlobKind;
use crate::block_builder::{BlockBuilder, HeaderBuilder, Metadata, OwnedBlock};
use crate::file_reader::FileReader;
use crate::file_writer::FileWriter;
use crate::writer::{Compression, PbfWriter};
pub(crate) type Result<T> = crate::BoxResult<T>;
pub(crate) fn flush_passthrough_buf(
chunks: &mut Vec<Vec<u8>>,
writer: &mut PbfWriter<FileWriter>,
) -> Result<()> {
if !chunks.is_empty() {
writer.write_raw_chunks(std::mem::take(chunks))?;
}
Ok(())
}
pub(crate) fn flush_block(bb: &mut BlockBuilder, writer: &mut PbfWriter<FileWriter>) -> Result<()> {
if let Some(OwnedBlock {
bytes,
index,
tagdata,
way_members,
}) = bb.take_owned()?
{
writer.write_primitive_block_owned(
bytes,
index,
tagdata.as_deref(),
way_members.as_deref(),
)?;
}
Ok(())
}
pub(crate) fn ensure_node_capacity(
bb: &mut BlockBuilder,
writer: &mut PbfWriter<FileWriter>,
) -> Result<()> {
if !bb.can_add_node() {
flush_block(bb, writer)?;
}
Ok(())
}
pub(crate) fn ensure_way_capacity(
bb: &mut BlockBuilder,
writer: &mut PbfWriter<FileWriter>,
) -> Result<()> {
if !bb.can_add_way() {
flush_block(bb, writer)?;
}
Ok(())
}
pub(crate) fn ensure_relation_capacity(
bb: &mut BlockBuilder,
writer: &mut PbfWriter<FileWriter>,
) -> Result<()> {
if !bb.can_add_relation() {
flush_block(bb, writer)?;
}
Ok(())
}
pub(crate) fn drain_batch_results<S>(
results: Vec<std::result::Result<(Vec<OwnedBlock>, S), String>>,
writer: &mut PbfWriter<FileWriter>,
mut merge: impl FnMut(S),
) -> Result<()> {
for result in results {
let (blocks, stats) = result.map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
merge(stats);
for OwnedBlock {
bytes: block_bytes,
index,
tagdata,
way_members,
} in blocks
{
writer.write_primitive_block_owned(
block_bytes,
index,
tagdata.as_deref(),
way_members.as_deref(),
)?;
}
}
Ok(())
}
pub(crate) fn flush_local(
bb: &mut BlockBuilder,
output: &mut Vec<OwnedBlock>,
) -> std::result::Result<(), String> {
if let Some(owned) = bb.take_owned().map_err(|e| e.to_string())? {
output.push(owned);
}
Ok(())
}
pub(crate) fn ensure_node_capacity_local(
bb: &mut BlockBuilder,
output: &mut Vec<OwnedBlock>,
) -> std::result::Result<(), String> {
if !bb.can_add_node() {
flush_local(bb, output)?;
}
Ok(())
}
pub(crate) fn ensure_way_capacity_local(
bb: &mut BlockBuilder,
output: &mut Vec<OwnedBlock>,
) -> std::result::Result<(), String> {
if !bb.can_add_way() {
flush_local(bb, output)?;
}
Ok(())
}
pub(crate) fn ensure_relation_capacity_local(
bb: &mut BlockBuilder,
output: &mut Vec<OwnedBlock>,
) -> std::result::Result<(), String> {
if !bb.can_add_relation() {
flush_local(bb, output)?;
}
Ok(())
}
pub(crate) fn warn_locations_on_ways_loss(header: &crate::HeaderBlock) {
if header.has_locations_on_ways()
|| header.has_way_members_v1()
|| header.has_shared_node_pins_v1()
{
eprintln!(
"Warning: input PBF has way-level enrichment metadata. \
LocationsOnWays and injected prepass metadata are not preserved in the output."
);
}
}
#[derive(Default)]
pub struct HeaderOverrides {
pub generator: Option<String>,
pub replication_timestamp: Option<i64>,
pub replication_sequence_number: Option<i64>,
pub replication_base_url: Option<String>,
}
impl HeaderOverrides {
pub fn parse(generator: Option<String>, output_headers: &[String]) -> Result<Self> {
let mut ov = HeaderOverrides {
generator,
..Default::default()
};
for entry in output_headers {
let (key, value) = entry.split_once('=').ok_or_else(|| {
format!("invalid --output-header format: '{entry}' (expected key=value)")
})?;
match key {
"osmosis_replication_timestamp" => {
ov.replication_timestamp = Some(value.parse::<i64>().map_err(|_| {
format!("invalid osmosis_replication_timestamp: '{value}'")
})?);
}
"osmosis_replication_sequence_number" => {
ov.replication_sequence_number = Some(value.parse::<i64>().map_err(|_| {
format!("invalid osmosis_replication_sequence_number: '{value}'")
})?);
}
"osmosis_replication_base_url" => {
ov.replication_base_url = Some(value.to_string());
}
_ => return Err(format!("unknown --output-header key: '{key}'").into()),
}
}
Ok(ov)
}
pub(crate) fn is_empty(&self) -> bool {
self.generator.is_none()
&& self.replication_timestamp.is_none()
&& self.replication_sequence_number.is_none()
&& self.replication_base_url.is_none()
}
pub(crate) fn apply<'a>(&'a self, mut hb: HeaderBuilder<'a>) -> HeaderBuilder<'a> {
if let Some(program) = &self.generator {
hb = hb.writing_program(program);
}
if let Some(ts) = self.replication_timestamp {
hb = hb.replication_timestamp(ts);
}
if let Some(seq) = self.replication_sequence_number {
hb = hb.replication_sequence_number(seq);
}
if let Some(url) = &self.replication_base_url {
hb = hb.replication_base_url(url);
}
hb
}
}
pub(crate) fn build_output_header(
header: &crate::HeaderBlock,
preserve_sorted: bool,
overrides: &HeaderOverrides,
configure: impl FnOnce(HeaderBuilder) -> HeaderBuilder,
) -> Result<Vec<u8>> {
let mut hb = configure(HeaderBuilder::from_header(header));
if preserve_sorted && header.is_sorted() {
hb = hb.sorted();
}
hb = overrides.apply(hb);
Ok(hb.build()?)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn writer_from_header(
output: &Path,
compression: Compression,
header: &crate::HeaderBlock,
preserve_sorted: bool,
overrides: &HeaderOverrides,
configure: impl FnOnce(HeaderBuilder) -> HeaderBuilder,
direct_io: bool,
io_uring: bool,
) -> Result<PbfWriter<FileWriter>> {
let header_bytes = build_output_header(header, preserve_sorted, overrides, configure)?;
writer_from_header_bytes(output, compression, &header_bytes, direct_io, io_uring)
}
pub(crate) fn writer_from_header_bytes(
output: &Path,
compression: Compression,
header_bytes: &[u8],
direct_io: bool,
io_uring: bool,
) -> Result<PbfWriter<FileWriter>> {
if io_uring {
#[cfg(feature = "linux-io-uring")]
{
Ok(PbfWriter::to_path_uring(output, compression, header_bytes)?)
}
#[cfg(not(feature = "linux-io-uring"))]
{
Err("--io-uring requires the linux-io-uring feature".into())
}
} else if direct_io {
#[cfg(feature = "linux-direct-io")]
{
Ok(PbfWriter::to_path_direct(
output,
compression,
header_bytes,
)?)
}
#[cfg(not(feature = "linux-direct-io"))]
{
Err("--direct-io requires the linux-direct-io feature".into())
}
} else {
Ok(PbfWriter::to_path(output, compression, header_bytes)?)
}
}
pub(crate) fn writer_from_header_bytes_parallel(
output: &Path,
compression: Compression,
header_bytes: &[u8],
direct_io: bool,
io_uring: bool,
) -> Result<PbfWriter<FileWriter>> {
if io_uring {
#[cfg(feature = "linux-io-uring")]
{
Ok(PbfWriter::to_path_uring(output, compression, header_bytes)?)
}
#[cfg(not(feature = "linux-io-uring"))]
{
Err("--io-uring requires the linux-io-uring feature".into())
}
} else if direct_io {
#[cfg(feature = "linux-direct-io")]
{
Ok(PbfWriter::to_path_direct(
output,
compression,
header_bytes,
)?)
}
#[cfg(not(feature = "linux-direct-io"))]
{
Err("--direct-io requires the linux-direct-io feature".into())
}
} else {
Ok(PbfWriter::to_path_parallel(
output,
compression,
header_bytes,
)?)
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn writer_from_header_parallel(
output: &Path,
compression: Compression,
header: &crate::HeaderBlock,
preserve_sorted: bool,
overrides: &HeaderOverrides,
configure: impl FnOnce(HeaderBuilder) -> HeaderBuilder,
direct_io: bool,
io_uring: bool,
) -> Result<PbfWriter<FileWriter>> {
let header_bytes = build_output_header(header, preserve_sorted, overrides, configure)?;
writer_from_header_bytes_parallel(output, compression, &header_bytes, direct_io, io_uring)
}
pub(crate) fn clean_metadata<'a>(
meta: Option<Metadata<'a>>,
clean: &cat::CleanAttrs,
) -> Option<Metadata<'a>> {
if !clean.any() {
return meta;
}
meta.map(|mut m| {
if clean.version {
m.version = 0;
}
if clean.changeset {
m.changeset = 0;
}
if clean.timestamp {
m.timestamp = 0;
}
if clean.uid {
m.uid = 0;
}
if clean.user {
m.user = "";
}
m
})
}
pub(crate) fn require_indexdata(
path: &Path,
direct_io: bool,
force: bool,
reason: &str,
) -> Result<bool> {
let present = has_indexdata(path, direct_io)?;
if !force && !present {
return Err(format!(
"{reason}\n\n\
Generate an indexed PBF first:\n\n\
\x20 pbfhogg cat input.osm.pbf -o indexed.osm.pbf\n\n\
Or pass --force to proceed anyway."
)
.into());
}
Ok(present)
}
pub(crate) fn require_sorted(
header: &crate::HeaderBlock,
path: &Path,
context: &str,
) -> Result<()> {
if !header.is_sorted() {
return Err(format!(
"{context} is not sorted (missing Sort.Type_then_ID optional feature).\n\
File: {}\n\n\
Sort the input file first:\n\n\
\x20 pbfhogg sort {} -o sorted.osm.pbf\n\n\
Streaming diff requires sorted inputs to operate in constant memory.",
path.display(),
path.display(),
)
.into());
}
Ok(())
}
pub(crate) fn require_sorted_err(path: &Path, context: &str) -> Result<()> {
Err(format!(
"{context} is not sorted (missing Sort.Type_then_ID optional feature).\n\
File: {}\n\n\
Sort the input file first:\n\n\
\x20 pbfhogg sort {} -o sorted.osm.pbf\n\n\
Streaming diff requires sorted inputs to operate in constant memory.",
path.display(),
path.display(),
)
.into())
}
pub(crate) fn dispatch_scan_arm(
input: &Path,
has_indexdata: bool,
min_blobs: u64,
) -> Result<crate::read::header_walker::ScanArm> {
use crate::read::header_walker::{ScanArm, choose_scan_arm_at, estimate_blob_count};
if !has_indexdata {
return Ok(ScanArm::Walker);
}
let estimate = estimate_blob_count(input)?;
crate::debug::emit_counter(
"walk_estimated_blobs",
i64::try_from(estimate.osmdata_blobs).unwrap_or(i64::MAX),
);
Ok(choose_scan_arm_at(&estimate, min_blobs))
}
pub fn has_indexdata(path: &Path, direct_io: bool) -> Result<bool> {
let mut reader = FileReader::open(path, direct_io)?;
let mut offset = 0u64;
while let Some(info) = crate::read::raw_frame::read_blob_header_only(&mut reader, &mut offset)?
{
if matches!(info.blob_type, BlobKind::OsmData) {
return Ok(info.index.is_some());
}
reader.skip(info.data_size as u64)?;
offset += info.data_size as u64;
}
Ok(false)
}
pub fn has_tagdata(path: &Path, direct_io: bool) -> Result<bool> {
let mut reader = FileReader::open(path, direct_io)?;
let mut offset = 0u64;
while let Some(info) = crate::read::raw_frame::read_blob_header_only(&mut reader, &mut offset)?
{
if matches!(info.blob_type, BlobKind::OsmData) {
return Ok(info.has_tagdata);
}
reader.skip(info.data_size as u64)?;
offset += info.data_size as u64;
}
Ok(false)
}
pub(crate) fn format_epoch_secs(secs: u64) -> String {
let secs = secs.cast_signed();
let day_secs = secs.rem_euclid(86400);
let days = (secs - day_secs) / 86400;
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
let h = day_secs / 3600;
let min = (day_secs % 3600) / 60;
let s = day_secs % 60;
format!("{y:04}-{m:02}-{d:02}T{h:02}:{min:02}:{s:02}Z")
}
pub fn parse_rfc3339_utc(input: &str) -> std::result::Result<i64, String> {
const FORMAT_ERR: &str = "timestamp must be YYYY-MM-DDTHH:MM:SSZ";
if input.len() != 20 {
return Err(FORMAT_ERR.to_owned());
}
let bytes = input.as_bytes();
if bytes[4] != b'-'
|| bytes[7] != b'-'
|| bytes[10] != b'T'
|| bytes[13] != b':'
|| bytes[16] != b':'
|| bytes[19] != b'Z'
{
return Err(FORMAT_ERR.to_owned());
}
let field = |start: usize, end: usize| -> std::result::Result<i64, String> {
input[start..end]
.parse::<i64>()
.map_err(|_| "invalid numeric timestamp component".to_owned())
};
let year = field(0, 4)?;
let month = field(5, 7)?;
let day = field(8, 10)?;
let hour = field(11, 13)?;
let minute = field(14, 16)?;
let second = field(17, 19)?;
if !(1..=12).contains(&month) {
return Err("invalid month in timestamp".to_owned());
}
if hour > 23 || minute > 59 || second > 59 {
return Err("invalid time in timestamp".to_owned());
}
let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
let max_day = match month {
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
4 | 6 | 9 | 11 => 30,
2 if leap => 29,
2 => 28,
_ => 0,
};
if day == 0 || day > max_day {
return Err("invalid day in timestamp".to_owned());
}
let y = year - i64::from(month <= 2);
let era = if y >= 0 { y } else { y - 399 } / 400;
let yoe = y - era * 400;
let mp = month + if month > 2 { -3 } else { 9 };
let doy = (153 * mp + 2) / 5 + day - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146_097 + doe - 719_468;
Ok(days * 86_400 + hour * 3_600 + minute * 60 + second)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rfc3339_parse_inverts_format() -> std::result::Result<(), String> {
assert_eq!(parse_rfc3339_utc("1970-01-01T00:00:00Z")?, 0);
assert_eq!(parse_rfc3339_utc("1970-01-02T00:00:00Z")?, 86_400);
assert_eq!(parse_rfc3339_utc("2004-02-29T12:00:00Z")?, 1_078_056_000);
for secs in [
0_u64,
951_827_696, 1_582_934_400, 1_771_622_445, 4_102_444_799, ] {
let formatted = format_epoch_secs(secs);
assert_eq!(
parse_rfc3339_utc(&formatted)?,
secs.cast_signed(),
"roundtrip failed for {formatted}"
);
}
assert!(parse_rfc3339_utc("2026-02-20 21:39:49").is_err()); assert!(parse_rfc3339_utc("2026-13-01T00:00:00Z").is_err()); assert!(parse_rfc3339_utc("2026-02-30T00:00:00Z").is_err()); assert!(parse_rfc3339_utc("2026-02-20T24:00:00Z").is_err()); assert!(parse_rfc3339_utc("not-a-timestamp-at-al").is_err());
Ok(())
}
}