pbfhogg 0.4.0

Fast OpenStreetMap PBF reader and writer for Rust. Read, write, and merge .osm.pbf files with pipelined parallel decoding.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

pub mod block_builder;
pub mod compression;
#[cfg(feature = "linux-direct-io")]
mod copy_range;
mod encode;
mod framing;
pub mod header_builder;
mod pipeline;
mod string_table;
#[cfg(feature = "linux-direct-io")]
pub mod direct_writer;
pub mod file_writer;
pub(crate) mod parallel_gzip;
pub(crate) mod parallel_writer;

// Under the `test-hooks` feature, expose the static fault-injection
// hooks so integration tests can arm them. The rest of
// `parallel_writer` stays crate-private.
#[cfg(feature = "test-hooks")]
pub mod parallel_writer_test_hooks {
    pub use super::parallel_writer::test_hooks::{
        PANIC_AT_POOL_OP_COUNT, POOL_OP_COUNT, reset,
    };
}

// Under the `test-hooks` feature, expose the parallel_gzip hooks and
// the `ParallelGzipWriter` type so integration tests can drive the
// writer directly.
#[cfg(feature = "test-hooks")]
pub mod parallel_gzip_test_hooks {
    pub use super::parallel_gzip::{ParallelGzipWriter, DEFAULT_CHUNK_SIZE};
    pub use super::parallel_gzip::test_hooks::{
        PANIC_AT_POOL_OP_COUNT, POOL_OP_COUNT, reset,
    };
}

// Under the `test-hooks` feature, expose the uring_writer hooks.
// Test arms `PANIC_AT_DISPATCH_COUNT` before invoking any code path
// that constructs a `PbfWriter::to_path_uring`.
#[cfg(all(feature = "test-hooks", feature = "linux-io-uring"))]
pub mod uring_writer_test_hooks {
    pub use super::uring_writer::test_hooks::{
        PANIC_AT_DISPATCH_COUNT, DISPATCH_COUNT, reset,
    };
}
#[cfg(feature = "linux-io-uring")]
pub mod uring_writer;
pub(crate) mod metrics;
pub(crate) mod raw_passthrough;
pub mod writer;

/// Page size for alignment. 4096 is universally safe across Linux filesystems.
#[cfg(any(feature = "linux-direct-io", feature = "linux-io-uring"))]
const PAGE_SIZE: usize = 4096;

/// Allocate `size` bytes aligned to `PAGE_SIZE`. Returns the pointer and layout
/// (needed for deallocation). Centralizes the unsafe page-aligned allocation
/// used by both `DirectWriter` and `AlignedBufferPool`.
#[cfg(any(feature = "linux-direct-io", feature = "linux-io-uring"))]
fn alloc_page_aligned(size: usize) -> std::io::Result<(std::ptr::NonNull<u8>, std::alloc::Layout)> {
    let layout = std::alloc::Layout::from_size_align(size, PAGE_SIZE)
        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
    // Safety: layout has non-zero size (callers pass BUF_CAPACITY or total > 0).
    let ptr = unsafe { std::alloc::alloc_zeroed(layout) };
    let ptr = std::ptr::NonNull::new(ptr)
        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::OutOfMemory, "aligned alloc failed"))?;
    Ok((ptr, layout))
}