use core::ops::Range;
use std::fs::File;
use std::io::{self, BufWriter, Seek as _, Write};
use std::path::Path;
use memmap2::Mmap;
use crate::error::{ArcsecError, Result};
pub const MAGIC: &[u8; 8] = b"ARCSECIX";
pub const VERSION: u32 = 1;
pub const EXTENSION: &str = "arcsecix";
const HEADER_LEN: usize = 256;
const BYTE_ORDER: u32 = 0x0A0B_0C0D;
const N_SECTIONS: usize = 5;
const SECTIONS_AT: usize = 72;
const HEADER_CRC_AT: usize = 252;
const TIER_LEN: usize = 40;
const STAR_LEN: usize = 12;
const QUAD_LEN: usize = 16;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct TierInfo {
pub radius: f64,
pub mag_cap: f32,
pub members: u32,
pub first_pattern: u64,
pub n_patterns: u64,
pub n_anchors: u64,
}
impl TierInfo {
#[must_use]
pub fn patterns(&self) -> Range<usize> {
self.first_pattern as usize..(self.first_pattern + self.n_patterns) as usize
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct IndexStar {
pub ra: f32,
pub dec: f32,
pub mag: i16,
pub tier: u8,
}
#[derive(Debug, Default)]
pub struct BuiltIndex {
pub tiers: Vec<TierInfo>,
pub stars: Vec<IndexStar>,
pub star_dir: Vec<u32>,
pub keys: Vec<u64>,
pub quads: Vec<[u32; 4]>,
pub source: String,
}
pub const STAR_BANDS: u32 = 720;
#[must_use]
pub fn star_band(dec: f64) -> u32 {
let b = ((dec + core::f64::consts::FRAC_PI_2) / core::f64::consts::PI * f64::from(STAR_BANDS))
.floor();
(b.max(0.0) as u32).min(STAR_BANDS - 1)
}
const fn crc_table() -> [u32; 256] {
let mut t = [0u32; 256];
let mut i = 0;
while i < 256 {
let mut c = i as u32;
let mut k = 0;
while k < 8 {
c = if c & 1 != 0 {
0xEDB8_8320 ^ (c >> 1)
} else {
c >> 1
};
k += 1;
}
t[i] = c;
i += 1;
}
t
}
static CRC_TABLE: [u32; 256] = crc_table();
#[derive(Clone, Copy)]
struct Crc(u32);
impl Crc {
fn new() -> Self {
Self(0xFFFF_FFFF)
}
fn update(&mut self, bytes: &[u8]) {
let mut c = self.0;
for &b in bytes {
c = CRC_TABLE[((c ^ u32::from(b)) & 0xFF) as usize] ^ (c >> 8);
}
self.0 = c;
}
fn finish(self) -> u32 {
!self.0
}
}
fn crc32(bytes: &[u8]) -> u32 {
let mut c = Crc::new();
c.update(bytes);
c.finish()
}
struct SectionWriter<W: Write> {
w: W,
pos: u64,
crc: Crc,
}
impl<W: Write> SectionWriter<W> {
fn put(&mut self, b: &[u8]) -> io::Result<()> {
self.w.write_all(b)?;
self.crc.update(b);
self.pos += b.len() as u64;
Ok(())
}
fn begin(&mut self) -> io::Result<u64> {
let pad = (8 - self.pos % 8) % 8;
self.w.write_all(&[0u8; 8][..pad as usize])?;
self.pos += pad;
self.crc = Crc::new();
Ok(self.pos)
}
}
impl BuiltIndex {
#[must_use]
pub fn file_size(&self) -> u64 {
(HEADER_LEN
+ self.tiers.len() * TIER_LEN
+ self.stars.len() * STAR_LEN
+ self.star_dir.len() * 4
+ self.keys.len() * 8
+ self.quads.len() * QUAD_LEN
+ 32) as u64
}
pub fn write(&self, path: &Path) -> Result<()> {
let tmp = path.with_extension(format!("{EXTENSION}.part"));
self.write_to(&tmp).map_err(ArcsecError::CatalogIo)?;
std::fs::rename(&tmp, path).map_err(ArcsecError::CatalogIo)
}
fn write_to(&self, path: &Path) -> io::Result<()> {
let f = File::create(path)?;
let mut sw = SectionWriter {
w: BufWriter::with_capacity(1 << 20, f),
pos: 0,
crc: Crc::new(),
};
sw.put(&[0u8; HEADER_LEN])?;
let mut sections = [(0u64, 0u64, 0u32); N_SECTIONS];
let off = sw.begin()?;
for t in &self.tiers {
sw.put(&t.radius.to_le_bytes())?;
sw.put(&t.mag_cap.to_le_bytes())?;
sw.put(&t.members.to_le_bytes())?;
sw.put(&t.first_pattern.to_le_bytes())?;
sw.put(&t.n_patterns.to_le_bytes())?;
sw.put(&t.n_anchors.to_le_bytes())?;
}
sections[0] = (off, sw.pos - off, sw.crc.finish());
let off = sw.begin()?;
for s in &self.stars {
let mut r = [0u8; STAR_LEN];
r[0..4].copy_from_slice(&s.ra.to_le_bytes());
r[4..8].copy_from_slice(&s.dec.to_le_bytes());
r[8..10].copy_from_slice(&s.mag.to_le_bytes());
r[10] = s.tier;
sw.put(&r)?;
}
sections[1] = (off, sw.pos - off, sw.crc.finish());
let off = sw.begin()?;
for &d in &self.star_dir {
sw.put(&d.to_le_bytes())?;
}
sections[2] = (off, sw.pos - off, sw.crc.finish());
let off = sw.begin()?;
for &k in &self.keys {
sw.put(&k.to_le_bytes())?;
}
sections[3] = (off, sw.pos - off, sw.crc.finish());
let off = sw.begin()?;
for q in &self.quads {
let mut r = [0u8; QUAD_LEN];
for (i, &s) in q.iter().enumerate() {
r[i * 4..i * 4 + 4].copy_from_slice(&s.to_le_bytes());
}
sw.put(&r)?;
}
sections[4] = (off, sw.pos - off, sw.crc.finish());
let mut h = [0u8; HEADER_LEN];
h[0..8].copy_from_slice(MAGIC);
h[8..12].copy_from_slice(&VERSION.to_le_bytes());
h[12..16].copy_from_slice(&BYTE_ORDER.to_le_bytes());
h[16..20].copy_from_slice(&(HEADER_LEN as u32).to_le_bytes());
h[20..24].copy_from_slice(&(super::pattern::BINS as u32).to_le_bytes());
h[24..28].copy_from_slice(&(self.tiers.len() as u32).to_le_bytes());
h[28..32].copy_from_slice(&STAR_BANDS.to_le_bytes());
h[32..40].copy_from_slice(&(self.stars.len() as u64).to_le_bytes());
h[40..48].copy_from_slice(&(self.keys.len() as u64).to_le_bytes());
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs() as i64);
h[48..56].copy_from_slice(&now.to_le_bytes());
let src = self.source.as_bytes();
let n = src.len().min(16);
h[56..56 + n].copy_from_slice(&src[..n]);
for (i, (o, l, c)) in sections.iter().enumerate() {
let at = SECTIONS_AT + i * 24;
h[at..at + 8].copy_from_slice(&o.to_le_bytes());
h[at + 8..at + 16].copy_from_slice(&l.to_le_bytes());
h[at + 16..at + 20].copy_from_slice(&c.to_le_bytes());
}
let crc = crc32(&h[..HEADER_CRC_AT]);
h[HEADER_CRC_AT..].copy_from_slice(&crc.to_le_bytes());
let mut f = sw.w.into_inner().map_err(io::IntoInnerError::into_error)?;
f.seek(io::SeekFrom::Start(0))?;
f.write_all(&h)?;
f.sync_all()
}
}
#[inline]
fn u32_at(b: &[u8], at: usize) -> u32 {
u32::from_le_bytes([b[at], b[at + 1], b[at + 2], b[at + 3]])
}
#[inline]
fn u64_at(b: &[u8], at: usize) -> u64 {
let mut x = [0u8; 8];
x.copy_from_slice(&b[at..at + 8]);
u64::from_le_bytes(x)
}
#[inline]
fn f32_at(b: &[u8], at: usize) -> f32 {
f32::from_bits(u32_at(b, at))
}
fn invalid(path: &Path, why: impl core::fmt::Display) -> ArcsecError {
ArcsecError::CatalogIo(io::Error::new(
io::ErrorKind::InvalidData,
format!("{}: not a usable arcsec blind index: {why}", path.display()),
))
}
pub struct BlindIndex {
map: Mmap,
tiers: Vec<TierInfo>,
sections: [(usize, usize, u32); N_SECTIONS],
n_stars: usize,
n_patterns: usize,
star_bands: u32,
source: String,
built_unix: i64,
}
impl core::fmt::Debug for BlindIndex {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("BlindIndex")
.field("source", &self.source)
.field("n_stars", &self.n_stars)
.field("n_patterns", &self.n_patterns)
.field("tiers", &self.tiers)
.finish_non_exhaustive()
}
}
#[must_use]
pub fn is_blind_index(path: &Path) -> bool {
use std::io::Read as _;
let mut m = [0u8; 8];
File::open(path)
.and_then(|mut f| f.read_exact(&mut m))
.is_ok()
&& &m == MAGIC
}
impl BlindIndex {
pub fn open(path: &Path) -> Result<Self> {
let file = File::open(path).map_err(ArcsecError::CatalogIo)?;
let map = unsafe { Mmap::map(&file) }.map_err(ArcsecError::CatalogIo)?;
if map.len() < HEADER_LEN || &map[..8] != MAGIC {
return Err(invalid(path, "bad magic"));
}
let version = u32_at(&map, 8);
if version != VERSION {
return Err(invalid(
path,
format!("format version {version}, this build reads {VERSION} - rebuild it"),
));
}
if u32_at(&map, 12) != BYTE_ORDER {
return Err(invalid(path, "written with the other byte order"));
}
if crc32(&map[..HEADER_CRC_AT]) != u32_at(&map, HEADER_CRC_AT) {
return Err(invalid(path, "header checksum mismatch"));
}
if u32_at(&map, 16) as usize != HEADER_LEN
|| f64::from(u32_at(&map, 20)) != super::pattern::BINS
{
return Err(invalid(
path,
"unsupported header length or descriptor bins",
));
}
let n_tiers = u32_at(&map, 24) as usize;
let star_bands = u32_at(&map, 28);
let n_stars = usize::try_from(u64_at(&map, 32)).map_err(|_| invalid(path, "too large"))?;
let n_patterns =
usize::try_from(u64_at(&map, 40)).map_err(|_| invalid(path, "too large"))?;
let built_unix = u64_at(&map, 48) as i64;
let source = String::from_utf8_lossy(&map[56..72])
.trim_end_matches('\0')
.to_string();
let mut sections = [(0usize, 0usize, 0u32); N_SECTIONS];
let expected = [
n_tiers.checked_mul(TIER_LEN),
n_stars.checked_mul(STAR_LEN),
(star_bands as usize + 1).checked_mul(4),
n_patterns.checked_mul(8),
n_patterns.checked_mul(QUAD_LEN),
];
for (i, s) in sections.iter_mut().enumerate() {
let at = SECTIONS_AT + i * 24;
let off = usize::try_from(u64_at(&map, at)).map_err(|_| invalid(path, "too large"))?;
let len =
usize::try_from(u64_at(&map, at + 8)).map_err(|_| invalid(path, "too large"))?;
if Some(len) != expected[i] || off.checked_add(len).is_none_or(|e| e > map.len()) {
return Err(invalid(
path,
format!("section {i} is truncated or mis-sized"),
));
}
*s = (off, len, u32_at(&map, at + 16));
}
let tb = &map[sections[0].0..sections[0].0 + sections[0].1];
let mut tiers = Vec::with_capacity(n_tiers);
for t in 0..n_tiers {
let r = &tb[t * TIER_LEN..(t + 1) * TIER_LEN];
let tier = TierInfo {
radius: f64::from_bits(u64_at(r, 0)),
mag_cap: f32_at(r, 8),
members: u32_at(r, 12),
first_pattern: u64_at(r, 16),
n_patterns: u64_at(r, 24),
n_anchors: u64_at(r, 32),
};
let end = tier.first_pattern.checked_add(tier.n_patterns);
if !(tier.radius.is_finite() && tier.radius > 0.0)
|| end.is_none_or(|e| e > n_patterns as u64)
{
return Err(invalid(path, format!("tier {t} is inconsistent")));
}
tiers.push(tier);
}
Ok(Self {
map,
tiers,
sections,
n_stars,
n_patterns,
star_bands,
source,
built_unix,
})
}
pub fn validate(&self) -> Result<()> {
const NAMES: [&str; N_SECTIONS] = ["tiers", "stars", "star directory", "keys", "quads"];
for (i, &(off, len, crc)) in self.sections.iter().enumerate() {
if crc32(&self.map[off..off + len]) != crc {
return Err(ArcsecError::CatalogIo(io::Error::new(
io::ErrorKind::InvalidData,
format!("blind index: {} section checksum mismatch", NAMES[i]),
)));
}
}
Ok(())
}
#[must_use]
pub fn tiers(&self) -> &[TierInfo] {
&self.tiers
}
#[must_use]
pub fn n_stars(&self) -> usize {
self.n_stars
}
#[must_use]
pub fn n_patterns(&self) -> usize {
self.n_patterns
}
#[must_use]
pub fn source(&self) -> &str {
&self.source
}
#[must_use]
pub fn built_unix(&self) -> i64 {
self.built_unix
}
#[must_use]
pub fn file_size(&self) -> usize {
self.map.len()
}
#[must_use]
pub fn tier_bytes(&self, t: &TierInfo) -> u64 {
t.n_patterns * (8 + QUAD_LEN as u64)
}
#[inline]
fn key(&self, i: usize) -> u64 {
u64_at(&self.map, self.sections[3].0 + i * 8)
}
#[must_use]
pub fn lookup(&self, tier: &TierInfo, key: u64) -> Range<usize> {
let r = tier.patterns();
let (mut lo, mut hi) = (r.start, r.end);
while lo < hi {
let mid = lo + (hi - lo) / 2;
if self.key(mid) < key {
lo = mid + 1;
} else {
hi = mid;
}
}
let start = lo;
let mut hi = r.end;
while lo < hi {
let mid = lo + (hi - lo) / 2;
if self.key(mid) <= key {
lo = mid + 1;
} else {
hi = mid;
}
}
start..lo
}
#[must_use]
pub fn quad(&self, i: usize) -> Option<[IndexStar; 4]> {
if i >= self.n_patterns {
return None;
}
let at = self.sections[4].0 + i * QUAD_LEN;
let mut out = [IndexStar {
ra: 0.0,
dec: 0.0,
mag: 0,
tier: 0,
}; 4];
for (k, s) in out.iter_mut().enumerate() {
*s = self.star(u32_at(&self.map, at + k * 4) as usize)?;
}
Some(out)
}
#[must_use]
pub fn star(&self, i: usize) -> Option<IndexStar> {
if i >= self.n_stars {
return None;
}
let at = self.sections[1].0 + i * STAR_LEN;
let b = &self.map[at..at + STAR_LEN];
Some(IndexStar {
ra: f32_at(b, 0),
dec: f32_at(b, 4),
mag: i16::from_le_bytes([b[8], b[9]]),
tier: b[10],
})
}
pub fn stars_near(&self, ra: f64, dec: f64, radius: f64, mut f: impl FnMut(&IndexStar)) {
use core::f64::consts::{FRAC_PI_2, PI};
if self.star_bands != STAR_BANDS {
return;
}
let lo_b = star_band((dec - radius).max(-FRAC_PI_2));
let hi_b = star_band((dec + radius).min(FRAC_PI_2));
let dir_at = self.sections[2].0;
for band in lo_b..=hi_b {
let s0 = u32_at(&self.map, dir_at + band as usize * 4) as usize;
let s1 =
(u32_at(&self.map, dir_at + (band as usize + 1) * 4) as usize).min(self.n_stars);
if s0 >= s1 {
continue;
}
let b_lo = f64::from(band) / f64::from(STAR_BANDS) * PI - FRAC_PI_2;
let b_hi = b_lo + PI / f64::from(STAR_BANDS);
let cos_min = b_lo.cos().min(b_hi.cos()).max(0.0);
let half = if cos_min * PI <= radius || dec.abs() + radius >= FRAC_PI_2 {
PI
} else {
(radius / cos_min).min(PI)
};
if half >= PI {
for i in s0..s1 {
if let Some(s) = self.star(i) {
f(&s);
}
}
continue;
}
let ra0 = (ra - half).rem_euclid(2.0 * PI);
let ra1 = (ra + half).rem_euclid(2.0 * PI);
let windows: &[(f64, f64)] = if ra0 <= ra1 {
&[(ra0, ra1)]
} else {
&[(ra0, 2.0 * PI), (0.0, ra1)]
};
for &(w0, w1) in windows {
let (mut lo, mut hi) = (s0, s1);
while lo < hi {
let mid = lo + (hi - lo) / 2;
let r = self.star(mid).map_or(f32::INFINITY, |s| s.ra);
if f64::from(r) < w0 {
lo = mid + 1;
} else {
hi = mid;
}
}
for i in lo..s1 {
let Some(s) = self.star(i) else { break };
if f64::from(s.ra) > w1 {
break;
}
f(&s);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::TempDir;
fn sample() -> BuiltIndex {
let stars: Vec<IndexStar> = (0..10)
.map(|i| IndexStar {
ra: 0.1 * i as f32,
dec: 0.2,
mag: 1000 + i,
tier: 0,
})
.collect();
let mut dir = vec![0u32; STAR_BANDS as usize + 1];
let b = star_band(0.2) as usize;
for (i, d) in dir.iter_mut().enumerate() {
*d = if i <= b { 0 } else { 10 };
}
BuiltIndex {
tiers: vec![TierInfo {
radius: 0.01,
mag_cap: 12.0,
members: 5,
first_pattern: 0,
n_patterns: 3,
n_anchors: 1,
}],
stars,
star_dir: dir,
keys: vec![5, 7, 7],
quads: vec![[0, 1, 2, 3], [1, 2, 3, 4], [5, 6, 7, 8]],
source: "d80".into(),
}
}
#[test]
fn round_trips_and_looks_up() {
let dir = TempDir::new("arcsecix_rt");
let p = dir.path().join("t.arcsecix");
sample().write(&p).unwrap();
assert!(is_blind_index(&p));
let ix = BlindIndex::open(&p).unwrap();
ix.validate().unwrap();
assert_eq!(ix.source(), "d80");
assert_eq!(ix.n_stars(), 10);
let t = ix.tiers()[0];
assert_eq!(ix.lookup(&t, 7), 1..3);
assert_eq!(ix.lookup(&t, 5), 0..1);
assert!(ix.lookup(&t, 6).is_empty());
assert_eq!(ix.quad(2).unwrap()[3].mag, 1008);
let mut n = 0;
ix.stars_near(0.3, 0.2, 0.11, |_| n += 1);
assert!((3..=5).contains(&n), "{n}");
}
#[test]
fn rejects_bad_magic_truncation_and_corruption() {
let dir = TempDir::new("arcsecix_bad");
let p = dir.path().join("t.arcsecix");
sample().write(&p).unwrap();
let good = std::fs::read(&p).unwrap();
let mut b = good.clone();
b[0] = b'X';
std::fs::write(&p, &b).unwrap();
assert!(BlindIndex::open(&p).is_err());
std::fs::write(&p, &good[..good.len() - 20]).unwrap();
assert!(BlindIndex::open(&p).is_err(), "truncated");
let mut b = good.clone();
b[30] ^= 1; std::fs::write(&p, &b).unwrap();
assert!(BlindIndex::open(&p).is_err());
let mut b = good.clone();
let n = b.len();
b[n - 3] ^= 0x40; std::fs::write(&p, &b).unwrap();
let ix = BlindIndex::open(&p).unwrap();
assert!(ix.validate().is_err());
}
#[test]
fn a_corrupt_star_reference_is_refused_not_followed() {
let dir = TempDir::new("arcsecix_ref");
let p = dir.path().join("t.arcsecix");
let mut s = sample();
s.quads[0] = [0, 1, 2, 999];
s.write(&p).unwrap();
let ix = BlindIndex::open(&p).unwrap();
assert!(ix.quad(0).is_none());
assert!(ix.quad(99).is_none());
}
#[test]
fn crc_matches_the_standard_check_value() {
assert_eq!(crc32(b"123456789"), 0xCBF4_3926);
}
}