use byteorder::{BigEndian, ReadBytesExt};
use log::{debug, warn};
use std::collections::BTreeMap;
use std::{
io,
io::{BufRead, BufReader, Read, Seek},
num::ParseIntError,
slice::ChunksExact,
str::{self, FromStr},
};
use super::{
super::skymap::{implicit::ImplicitSkyMapArray, SkyMapEnum},
error::FitsError,
gz::{is_gz, uncompress},
keywords::{
CoordSys, FitsCard, IndexSchema, Nside, Order, Ordering, SkymapKeywords, SkymapKeywordsMap,
TForm, TForm1, TForm2, TType1,
},
};
use crate::nested::map::skymap::explicit::ExplicitSkyMapBTree;
use crate::{depth, is_nside, n_hash};
pub fn from_fits_skymap<R: Read + Seek>(mut reader: BufReader<R>) -> Result<SkyMapEnum, FitsError> {
if is_gz(&mut reader)? {
from_fits_skymap_internal(uncompress(reader))
} else {
from_fits_skymap_internal(reader)
}
}
pub fn from_fits_skymap_internal<R: BufRead>(mut reader: R) -> Result<SkyMapEnum, FitsError> {
let mut header_block = [b' '; 2880];
debug!("Parse primary HDU...");
consume_primary_hdu(&mut reader, &mut header_block)?;
debug!("Parse extension...");
let mut it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?;
check_keyword_and_val(it80.next().unwrap(), b"XTENSION", b"'BINTABLE'")?;
check_keyword_and_val(it80.next().unwrap(), b"BITPIX ", b"8")?;
check_keyword_and_val(it80.next().unwrap(), b"NAXIS ", b"2")?;
let n_bytes_per_row = check_keyword_and_parse_uint_val::<u64>(it80.next().unwrap(), b"NAXIS1 ")?;
let n_rows = check_keyword_and_parse_uint_val::<u64>(it80.next().unwrap(), b"NAXIS2 ")?;
check_keyword_and_val(it80.next().unwrap(), b"PCOUNT ", b"0")?;
check_keyword_and_val(it80.next().unwrap(), b"GCOUNT ", b"1")?;
let n_cols = check_keyword_and_parse_uint_val::<u64>(it80.next().unwrap(), b"TFIELDS ")?;
let mut skymap_kws = SkymapKeywordsMap::new();
'hr: loop {
for kw_record in &mut it80 {
match SkymapKeywords::is_skymap_kw(kw_record)? {
Some(kw) => {
if let Some(previous_mkw) = skymap_kws.insert(kw) {
warn!(
"Keyword '{}' found more than once in a same HDU! We use the first occurrence.",
previous_mkw.keyword_str()
);
skymap_kws.insert(previous_mkw);
}
}
None => {
if &kw_record[0..4] == b"END " {
break 'hr;
} else {
debug!("Ignored FITS card: {}", unsafe {
str::from_utf8_unchecked(kw_record)
})
}
}
}
}
it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?;
}
match skymap_kws.get::<TType1>() {
Some(SkymapKeywords::TType1(ttype1)) => debug!("Skymap column name: {}", ttype1.get()),
None => warn!("Missing keyword {}.", TType1::keyword_str()),
_ => unreachable!(),
};
let coltype_1 = match skymap_kws.get::<TForm1>() {
Some(SkymapKeywords::TForm1(tform1)) => Ok(tform1.to_tform()),
None => Err(FitsError::MissingKeyword {
keyword: TForm1::keyword_string(),
}),
_ => unreachable!(),
}?;
debug!("Skymap column type: {}", coltype_1.to_fits_value());
skymap_kws.check_pixtype()?; skymap_kws.check_ordering(Ordering::Nested, false)?;
skymap_kws.check_coordsys(CoordSys::Cel, true)?;
let depth = match skymap_kws.get::<Order>() {
Some(SkymapKeywords::Order(order)) => Ok(order.get()),
None => match skymap_kws.get::<Nside>() {
Some(SkymapKeywords::Nside(nside)) => {
let nside = nside.get();
if is_nside(nside) {
Ok(depth(nside))
} else {
Err(FitsError::new_custom(format!(
"Nside is not valid (to be used in nested mode at least): {}",
nside
)))
}
}
None => Err(FitsError::new_custom(String::from(
"Both keywords 'ORDER' and 'NSIDE' are missing!",
))),
_ => unreachable!(),
},
_ => unreachable!(),
}?;
match skymap_kws.get::<IndexSchema>() {
Some(SkymapKeywords::IndexSchema(IndexSchema::Implicit)) | None => {
if n_cols != 1 {
return Err(FitsError::UnexpectedValue {
keyword: String::from("TFIELDS"),
expected: 1.to_string(),
actual: n_cols.to_string(),
});
}
skymap_kws.check_firstpix(0, true)?;
let n_hash = n_hash(depth);
skymap_kws.check_lastpix(n_hash - 1, true)?;
let n_hash_2 = coltype_1.n_pack() as u64 * n_rows;
if n_hash != n_hash_2 {
return Err(FitsError::new_custom(format!(
"Number of elements {} do not match number of HEALPix cells {}",
n_hash_2, n_hash
)));
}
if n_bytes_per_row != coltype_1.n_bytes() as u64 {
return Err(FitsError::new_custom(format!(
"Number of bytes per row {} do not match TFORM1 = {}",
n_bytes_per_row,
coltype_1.to_fits_value()
)));
}
match coltype_1 {
TForm::B(_) => (0..n_hash)
.map(|_| reader.read_u8())
.collect::<Result<Vec<u8>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64U8(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
TForm::I(_) => (0..n_hash)
.map(|_| reader.read_i16::<BigEndian>())
.collect::<Result<Vec<i16>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64I16(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
TForm::J(_) => (0..n_hash)
.map(|_| reader.read_i32::<BigEndian>())
.collect::<Result<Vec<i32>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64I32(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
TForm::K(_) => (0..n_hash)
.map(|_| reader.read_i64::<BigEndian>())
.collect::<Result<Vec<i64>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64I64(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
TForm::E(_) => (0..n_hash)
.map(|_| reader.read_f32::<BigEndian>())
.collect::<Result<Vec<f32>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64F32(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
TForm::D(_) => (0..n_hash)
.map(|_| reader.read_f64::<BigEndian>())
.collect::<Result<Vec<f64>, io::Error>>()
.map(|v| {
SkyMapEnum::ImplicitU64F64(ImplicitSkyMapArray::new(depth, v.into_boxed_slice()))
}),
}
.map_err(FitsError::Io)
}
Some(SkymapKeywords::IndexSchema(IndexSchema::Explicit)) => {
let coltype_2 = match skymap_kws.get::<TForm2>() {
Some(SkymapKeywords::TForm2(tform2)) => Ok(tform2.to_tform()),
None => Err(FitsError::MissingKeyword {
keyword: TForm1::keyword_string(),
}),
_ => unreachable!(),
}?;
if n_cols != 2 {
return Err(FitsError::UnexpectedValue {
keyword: String::from("TFIELDS"),
expected: 2.to_string(),
actual: n_cols.to_string(),
});
}
if n_bytes_per_row != (coltype_1.n_bytes() + coltype_2.n_bytes()) as u64 {
return Err(FitsError::new_custom(format!(
"Number of bytes per row {} do not match TFORM1 = {} plus FTORM2 = {}",
n_bytes_per_row,
coltype_1.to_fits_value(),
coltype_2.to_fits_value()
)));
}
match (coltype_1, coltype_2) {
(TForm::J(_), TForm::B(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_u8().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, u8>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32U8(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::J(_), TForm::I(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_i16::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, i16>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32I16(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::J(_), TForm::J(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_i32::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, i32>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32I32(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::J(_), TForm::K(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_i64::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, i64>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32I64(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::J(_), TForm::E(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_f32::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, f32>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32F32(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::J(_), TForm::D(_)) => (0..n_rows)
.map(|_| {
reader
.read_u32::<BigEndian>()
.and_then(|k| reader.read_f64::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u32, f64>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU32F64(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::B(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_u8().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, u8>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64U8(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::I(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_i16::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, i16>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64I16(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::J(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_i32::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, i32>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64I32(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::K(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_i64::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, i64>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64I64(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::E(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_f32::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, f32>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64F32(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
(TForm::K(_), TForm::D(_)) => (0..n_rows)
.map(|_| {
reader
.read_u64::<BigEndian>()
.and_then(|k| reader.read_f64::<BigEndian>().map(|v| (k, v)))
})
.collect::<Result<BTreeMap<u64, f64>, io::Error>>()
.map(|v| SkyMapEnum::ExplicitU64F64(ExplicitSkyMapBTree::new(depth, v)))
.map_err(FitsError::Io),
_ => Err(FitsError::UnexpectedValue {
keyword: String::from("TFORM1"),
expected: String::from("J or K"),
actual: coltype_2.to_fits_value(),
}),
}
}
Some(SkymapKeywords::IndexSchema(IndexSchema::Sparse)) => Err(FitsError::UnexpectedValue {
keyword: IndexSchema::keyword_string(),
expected: format!(
"{} or {}",
IndexSchema::Implicit.to_fits_value(),
IndexSchema::Explicit.to_fits_value()
),
actual: IndexSchema::Sparse.to_fits_value(),
}),
Some(_) => unreachable!(),
}
}
const VALUE_INDICATOR: &[u8; 2] = b"= ";
fn consume_primary_hdu<R: BufRead>(
reader: &mut R,
header_block: &mut [u8; 2880],
) -> Result<(), FitsError> {
let mut chunks_of_80 = next_36_chunks_of_80_bytes(reader, header_block)?;
check_keyword_and_val(chunks_of_80.next().unwrap(), b"SIMPLE ", b"T")?;
chunks_of_80.next().unwrap();
match check_keyword_and_val(chunks_of_80.next().unwrap(), b"NAXIS ", b"0") {
Ok(()) => Ok(()),
Err(FitsError::UnexpectedValue { .. }) => {
check_keyword_and_val(chunks_of_80.next().unwrap(), b"NAXIS1 ", b"0")
}
Err(e) => Err(e),
}?;
while !contains_end(&mut chunks_of_80) {
chunks_of_80 = next_36_chunks_of_80_bytes(reader, header_block)?;
}
Ok(())
}
pub(crate) fn next_36_chunks_of_80_bytes<'a, R: Read>(
reader: &'a mut R,
header_block: &'a mut [u8; 2880],
) -> Result<ChunksExact<'a, u8>, FitsError> {
reader
.read_exact(header_block)
.map_err(FitsError::Io)
.map(|()| header_block.chunks_exact(80))
}
fn contains_end<'a, I: Iterator<Item = &'a [u8]>>(chunks_of_80: &'a mut I) -> bool {
for kw_rc in chunks_of_80 {
debug_assert_eq!(kw_rc.len(), 80);
if &kw_rc[0..4] == b"END " {
return true;
}
}
false
}
pub(crate) fn check_keyword_and_val(
keyword_record: &[u8],
expected_kw: &[u8],
expected_val: &[u8],
) -> Result<(), FitsError> {
debug!(
"Check KW: '{}' and val: '{}' in card: '{}'.",
unsafe { str::from_utf8_unchecked(expected_kw) },
unsafe { str::from_utf8_unchecked(expected_val) },
unsafe { str::from_utf8_unchecked(keyword_record) }
);
check_expected_keyword(keyword_record, expected_kw)
.and_then(|()| check_for_value_indicator(keyword_record))
.and_then(|()| check_expected_value(keyword_record, expected_val))
}
pub(crate) fn check_keyword_and_str_val(
keyword_record: &[u8],
expected_kw: &[u8],
expected_val: &[u8],
) -> Result<(), FitsError> {
debug!(
"Check KW: '{}' and val: '{}' in card: '{}'.",
unsafe { str::from_utf8_unchecked(expected_kw) },
unsafe { str::from_utf8_unchecked(expected_val) },
unsafe { str::from_utf8_unchecked(keyword_record) }
);
check_expected_keyword(keyword_record, expected_kw)
.and_then(|()| check_for_value_indicator(keyword_record))
.and_then(|()| check_expected_str_value(keyword_record, expected_val))
}
pub(crate) fn check_keyword_and_parse_uint_val<T>(
keyword_record: &[u8],
expected_kw: &[u8],
) -> Result<T, FitsError>
where
T: Into<u64> + FromStr<Err = ParseIntError>,
{
debug!(
"Check KW: '{}' and return uint from card: '{}'.",
unsafe { str::from_utf8_unchecked(expected_kw) },
unsafe { str::from_utf8_unchecked(keyword_record) }
);
check_expected_keyword(keyword_record, expected_kw)
.and_then(|()| check_for_value_indicator(keyword_record))
.and_then(|()| parse_uint_val::<T>(keyword_record))
}
#[allow(dead_code)]
pub(crate) fn check_keyword_and_get_str_val<'a>(
keyword_record: &'a [u8],
expected_kw: &[u8],
) -> Result<&'a str, FitsError> {
debug!(
"Check KW: '{}' and return str from card: '{}'.",
unsafe { str::from_utf8_unchecked(expected_kw) },
unsafe { str::from_utf8_unchecked(keyword_record) }
);
check_expected_keyword(keyword_record, expected_kw)
.and_then(|()| check_for_value_indicator(keyword_record))
.and_then(|()| {
get_str_val_no_quote(keyword_record).map(|bytes| unsafe { str::from_utf8_unchecked(bytes) })
})
}
pub(crate) fn check_expected_keyword(
keyword_record: &[u8],
expected: &[u8],
) -> Result<(), FitsError> {
debug_assert!(keyword_record.len() == 80); debug_assert!(expected.len() <= 8); if &keyword_record[..expected.len()] == expected {
Ok(())
} else {
let expected = String::from(unsafe { str::from_utf8_unchecked(expected) }.trim_end());
let actual = String::from_utf8_lossy(&keyword_record[..expected.len()])
.trim_end()
.to_string();
Err(FitsError::UnexpectedKeyword { expected, actual })
}
}
pub(crate) fn check_for_value_indicator(keyword_record: &[u8]) -> Result<(), FitsError> {
debug_assert!(keyword_record.len() == 80); if get_value_indicator(keyword_record) == VALUE_INDICATOR {
Ok(())
} else {
let keyword_record = String::from_utf8_lossy(keyword_record)
.trim_end()
.to_string();
Err(FitsError::ValueIndicatorNotFound { keyword_record })
}
}
pub(crate) fn get_keyword(keyword_record: &[u8]) -> &[u8] {
&keyword_record[..8]
}
pub(crate) fn get_value_indicator(keyword_record: &[u8]) -> &[u8] {
&keyword_record[8..10]
}
pub(crate) fn get_value(keyword_record: &[u8]) -> &[u8] {
&keyword_record[10..]
}
pub(crate) fn get_left_trimmed_value(keyword_record: &[u8]) -> &[u8] {
get_value(keyword_record).trim_ascii_start()
}
pub(crate) fn check_expected_value(
keyword_record: &[u8],
expected: &[u8],
) -> Result<(), FitsError> {
debug_assert!(keyword_record.len() == 80); let src = get_value(keyword_record);
let lt_src = src.trim_ascii_start();
if lt_src.len() >= expected.len() && <_src[..expected.len()] == expected {
Ok(())
} else {
let keyword = String::from_utf8_lossy(&keyword_record[0..8])
.trim_end()
.to_string();
let expected = String::from(unsafe { str::from_utf8_unchecked(expected) });
let actual = String::from_utf8_lossy(<_src[..expected.len()]).to_string();
Err(FitsError::UnexpectedValue {
keyword,
expected,
actual,
})
}
}
pub(crate) fn check_expected_str_value(
keyword_record: &[u8],
expected: &[u8],
) -> Result<(), FitsError> {
debug_assert!(keyword_record.len() == 80); get_str_val_no_quote(keyword_record).and_then(|actual| {
if actual == expected {
Ok(())
} else {
let keyword = String::from_utf8_lossy(&keyword_record[0..8])
.trim_end()
.to_string();
let expected = String::from(unsafe { str::from_utf8_unchecked(expected) });
let actual = String::from_utf8_lossy(actual).to_string();
Err(FitsError::UnexpectedValue {
keyword,
expected,
actual,
})
}
})
}
pub(crate) fn get_str_val_no_quote(keyword_record: &[u8]) -> Result<&[u8], FitsError> {
let mut it = get_left_trimmed_value(keyword_record).split_inclusive(|c| *c == b'\'');
if let Some([b'\'']) = it.next() {
if let Some([subslice @ .., b'\'']) = it.next() {
return Ok(subslice.trim_ascii());
}
}
let keyword_record = String::from_utf8_lossy(keyword_record)
.trim_end()
.to_string();
Err(FitsError::StringValueNotFound { keyword_record })
}
pub(crate) fn parse_uint_val<T>(keyword_record: &[u8]) -> Result<T, FitsError>
where
T: Into<u64> + FromStr<Err = ParseIntError>,
{
let src = get_left_trimmed_value(keyword_record);
let to = index_of_last_digit(src);
if to == 0 {
let keyword_record = String::from_utf8_lossy(keyword_record)
.trim_end()
.to_string();
Err(FitsError::UintValueNotFound { keyword_record })
} else {
let str_val = unsafe { str::from_utf8_unchecked(&src[..to]) };
str_val.parse::<T>().map_err(|e| FitsError::WrongUintValue {
context: str_val.to_string(),
err: e,
})
}
}
pub(crate) fn index_of_last_digit(src: &[u8]) -> usize {
for (i, c) in src.iter().enumerate() {
if !c.is_ascii_digit() {
return i;
}
}
src.len()
}