use crate::averager::{AverageFrame, FrameParams};
use crate::results::{ResultsActor, TaskToDo};
use crate::utils::{
py_stamp, BubbleError, BubbleResult, Compressor, Decompressor, Resizer, Texter,
KEY_BUBBLE_CAKE, KEY_DONT_TOUCH, KEY_DUBBLE_MONITOR, KEY_DUBBLE_PHOTO,
};
use crate::{dispatcher::IType, Shutdown};
use actix::{Actor, ActorContext, Addr, Handler, Message, Running, SyncContext, System};
use cryiorust::edf::{DataType, Edf};
use cryiorust::frame::{self, Frame, Header, HeaderEntry};
use cryiorust::{cbf, edf};
use integrustio::distortion::Distortion;
use integrustio::integrator::{
Diffractogram, Integrable, IntegrationType, Integrator, Pattern, PatternType, Units,
};
use integrustio::poni::Poni;
use integrustio::spline::Spline;
use itertools::Itertools;
use parking_lot::{RwLock, RwLockUpgradableReadGuard};
use serde;
use std::fmt::Debug;
use std::fs::{self, metadata, File};
use std::io::{self, BufWriter, Cursor, Write};
#[cfg(not(target_os = "windows"))]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::{fmt, mem, thread};
use uuid::Uuid;
#[derive(serde::Deserialize, Debug)]
pub struct Settings {
#[serde(default)]
run: i32,
#[serde(default)]
stop: i32,
#[serde(default)]
path: Option<String>,
#[serde(default)]
poni: Option<String>,
#[serde(default)]
subdir: Option<String>,
#[serde(default, rename = "solidangle")]
solid_angle: Option<bool>,
#[serde(default)]
polarization: Option<f64>,
#[serde(default)]
calibration: Option<f64>,
#[serde(default)]
units: Option<String>,
#[serde(default)]
normalization: Option<String>,
#[serde(default, rename = "normrange")]
norm_range: Option<Vec<f64>>,
#[serde(default)]
beamline: Option<String>,
#[serde(default, rename = "bins")]
radial_bins: Option<usize>,
#[serde(default, rename = "abins")]
azimuthal_bins: Option<usize>,
#[serde(default)]
thickness: Option<f64>,
#[serde(default)]
concentration: Option<f64>,
#[serde(default, rename = "ext")]
extension: Option<String>,
#[serde(default)]
speed: Option<bool>,
#[serde(default)]
super_speed: Option<bool>,
#[serde(default, rename = "bkgCoef")]
bkg_coef: Option<f64>,
#[serde(default, rename = "backgroundFiles")]
bkg_names: Option<Vec<String>>,
#[serde(default)]
detector: Option<String>,
#[serde(default, rename = "darkBackground")]
dark_bkg_names: Option<Vec<String>>,
#[serde(default, rename = "darkSample")]
dark_sample_names: Option<Vec<String>>,
#[serde(default)]
spline: Option<String>,
#[serde(default, rename = "maskFile")]
mask: Option<String>,
#[serde(default)]
flood: Option<String>,
#[serde(default, rename = "save")]
save_edf: Option<bool>,
#[serde(default)]
_incidence: Option<bool>,
#[serde(default, rename = "multicolumnName")]
multi_column: Option<String>,
#[serde(default, rename = "azimuthChecked")]
use_azimuth: Option<bool>,
#[serde(default)]
azimuth: Option<Vec<f64>>,
#[serde(default, rename = "useRadial")]
use_radial: Option<bool>,
#[serde(default)]
radial: Option<Vec<f64>>,
#[serde(default, rename = "azimuthSlices")]
azimuth_slices: Option<bool>,
#[serde(default, rename = "i_azimuth")]
azimuthal: Option<bool>,
#[serde(default, rename = "i_cake")]
cake: Option<bool>,
#[serde(default, rename = "averageNFrames")]
average: Option<usize>,
}
#[derive(PartialEq)]
enum Detector {
Pilatus,
Frelon,
}
impl Detector {
fn is_frelon(&self) -> bool {
self == &Detector::Frelon
}
}
pub struct IntegrationActor {
i: Arc<RwLock<Integration>>,
results: Addr<ResultsActor>,
}
impl IntegrationActor {
pub fn new(i: Arc<RwLock<Integration>>, results: Addr<ResultsActor>) -> IntegrationActor {
IntegrationActor { i, results }
}
fn integrate(&mut self, frame: &mut Box<dyn Frame>, path: Arc<PathBuf>) -> bool {
let results = {
let i = self.i.upgradable_read();
if !i.is_running() {
return false;
}
match i.integrate(frame) {
Some(results) => results,
None => RwLockUpgradableReadGuard::upgrade(i).init(frame),
}
};
let frame_number = self.frame_number(&frame, &path);
for mut result in results {
result.path = path.clone();
if let Err(err) = result.save(frame_number) {
error!("Failed to save: {}", err);
}
self.results.do_send(result);
}
true
}
fn frame_number(&self, frame: &Box<dyn Frame>, path: &PathBuf) -> Option<usize> {
if frame.is_multi() {
let frame_number = frame.current_frame();
debug!("Processing frame {} in {:?}", frame_number, path);
Some(frame_number)
} else {
None
}
}
fn average<T: AsRef<Path> + fmt::Debug>(
&self,
mut frame: &mut Box<dyn Frame>,
average: usize,
beamline: &Beamline,
path: T,
) -> Option<Box<dyn Frame>> {
if average > 1 && frame.is_multi() {
let fp = beamline.monitor_and_transmission(&frame);
debug!(
"Averaging frame {}(1) out of {} from {:?}",
frame.current_frame(),
average,
&path
);
let mut af = Box::new(AverageFrame::new(&mut frame, fp));
for n in 1..average {
if !self.i.read().is_running() {
return None;
}
match frame.next_frame() {
Ok(_) => match af.append(&frame, &beamline.monitor_and_transmission(&frame)) {
Ok(_) => debug!(
"Averaging frame {}({}) out of {} from {:?}",
frame.current_frame(),
n + 1,
average,
&path
),
Err(err) => {
error!("Error while averaging multi-frame {:?}: {}", path, err);
return None;
}
},
Err(_) => return Some(af),
}
}
}
None
}
fn parse_frame(&mut self, mut frame: Box<dyn Frame>, path: PathBuf) {
let path = Arc::new(path);
let (beamline, average) = {
let i = self.i.read();
if !i.is_running() {
return;
} else {
(i.beamline, i.average)
}
};
loop {
if !self.integrate(
match self.average(&mut frame, average, &beamline, path.as_ref()) {
None => &mut frame,
Some(ref mut frame) => frame,
},
path.clone(),
) {
return;
}
if let Err(_) = frame.next_frame() {
return;
}
}
}
}
impl Actor for IntegrationActor {
type Context = SyncContext<Self>;
fn started(&mut self, _: &mut Self::Context) {
debug!(
"{} Integration has started in thread {:?}",
self.i.read().itype,
thread::current().id()
);
}
fn stopping(&mut self, _: &mut Self::Context) -> Running {
debug!(
"{} Integration is stopping in {:?}",
self.i.read().itype,
thread::current().id()
);
Running::Stop
}
fn stopped(&mut self, _: &mut Self::Context) {
debug!(
"{} Integration has stopped in {:?}",
self.i.read().itype,
thread::current().id()
);
System::current().stop();
}
}
impl Handler<Shutdown> for IntegrationActor {
type Result = ();
fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result {
ctx.stop();
}
}
pub struct Integration {
i: Integrator,
subdir: Arc<PathBuf>,
itype: IType,
running: bool,
path: String,
beamline: Beamline,
thickness: f64,
concentration: f64,
extension: Arc<String>,
speed: bool,
super_speed: bool,
detector: Detector,
dark_bkg: Option<AverageFrame>,
dark_sample: Option<AverageFrame>,
spline: Option<Spline>,
d: Distortion,
r_bins: usize,
a_bins: usize,
save_edf: bool,
flood: Option<Box<dyn Frame>>,
norm: Normalization,
norm_range: [f64; 2],
calibration: f64,
bkg: Option<AverageFrame>,
multi_column: Arc<String>,
azimuth: [f64; 2],
user_azimuth: [f64; 2],
radial: [f64; 2],
azimuth_slices: bool,
azimuthal: bool,
cake: bool,
mask: Option<Vec<u8>>,
average: usize,
}
#[derive(Copy, Clone)]
pub enum Beamline {
None,
Dubble,
SNBL,
}
impl Integration {
pub fn new(itype: IType) -> Integration {
Integration {
itype,
running: false,
path: String::new(),
i: Integrator::new(),
subdir: Arc::new(PathBuf::new()),
beamline: Beamline::None,
thickness: 0.0,
concentration: 0.0,
extension: Arc::new("dat".to_string()),
speed: false,
super_speed: false,
detector: Detector::Pilatus,
dark_bkg: None,
dark_sample: None,
spline: None,
d: Distortion::new(),
r_bins: 0,
a_bins: 0,
save_edf: false,
flood: None,
norm: Normalization::None,
norm_range: [0., 0.],
calibration: 1.0,
bkg: None,
multi_column: Arc::new(String::new()),
azimuth: [0., 0.],
user_azimuth: [0., 0.],
radial: [0., 0.],
azimuth_slices: false,
azimuthal: false,
cake: false,
mask: None,
average: 0,
}
}
pub fn itype(&self) -> &IType {
&self.itype
}
pub fn average(&self) -> usize {
self.average
}
pub fn path(&self) -> &str {
&self.path
}
pub fn multi_column(&self) -> &str {
&self.multi_column
}
pub fn extension(&self) -> &str {
&self.extension
}
pub fn parse_settings(&mut self, s: Settings) -> (Vec<String>, Vec<String>) {
let mut errors = vec![];
let mut warnings = vec![];
if let Some(err) = self.set_path(s.path, s.subdir) {
errors.push(err);
}
if let Some(err) = self.set_poni(s.poni) {
errors.push(err);
}
self.set_solid_angle(s.solid_angle);
self.set_units(s.units);
self.set_polarization(s.polarization);
self.set_calibration(s.calibration);
self.set_beamline(s.beamline);
self.set_radial_bins(s.radial_bins);
self.set_azimuthal_bins(s.azimuthal_bins);
self.set_thickness(s.thickness);
self.set_average(s.average);
self.set_concentration(s.concentration);
self.set_extension(s.extension);
self.set_speed(s.speed);
self.set_super_speed(s.super_speed);
self.set_save_edf(s.save_edf);
self.set_multi_column(s.multi_column);
self.set_azimuth_slices(s.azimuth_slices);
self.set_azimuth(s.azimuth, s.use_azimuth);
self.set_radial(s.radial, s.use_radial);
self.set_azimuthal_integration(s.azimuthal);
self.set_cake(s.cake);
if let Some(err) = self.set_normalization(s.normalization, s.norm_range) {
warnings.push(err);
}
if let Some(err) = self.set_mask(s.mask) {
warnings.push(err);
}
if let Some(err) = self.set_detector(s.detector) {
errors.push(err);
}
if let Some(err) = self.parse_spline(s.spline) {
warnings.push(err);
}
if let Some(mut errs) = self.set_dark_background(s.dark_bkg_names) {
warnings.append(&mut errs);
}
if let Some(mut errs) = self.set_dark_sample(s.dark_sample_names) {
warnings.append(&mut errs);
}
if let Some(err) = self.set_flood(s.flood) {
warnings.push(err);
}
if let Some(mut errs) = self.set_background(s.bkg_coef, s.bkg_names) {
warnings.append(&mut errs);
}
if s.run != 0 && errors.is_empty() {
self.run();
}
if s.stop != 0 {
self.stop();
}
(warnings, errors)
}
fn convert_azimuth(&self, azimuth: &[f64; 2]) -> [f64; 2] {
let mut out = match self.beamline {
Beamline::SNBL => [-(azimuth[1] + 180.), -(azimuth[0] + 180.)],
Beamline::Dubble => match self.itype {
IType::SAXS => azimuth.clone(),
IType::WAXS => [azimuth[0] - 180., azimuth[1] - 180.],
},
Beamline::None => azimuth.clone(),
};
while out[0] < 0. {
out[0] += 360.;
}
while out[0] >= 360. {
out[0] -= 360.;
}
while out[1] <= 0. {
out[1] += 360.;
}
while out[1] > 360. {
out[1] -= 360.;
}
out
}
fn set_azimuth(&mut self, azimuth: Option<Vec<f64>>, enable: Option<bool>) {
if let Some(enable) = enable {
if enable {
match azimuth {
None => (),
Some(azimuth) => {
if azimuth.len() < 2 {
self.user_azimuth = [0., 0.];
self.azimuth = self.user_azimuth.clone();
return;
}
self.user_azimuth =
[azimuth[0].min(azimuth[1]), azimuth[1].max(azimuth[0])];
self.azimuth = if self.azimuth_slices {
self.user_azimuth.clone()
} else {
self.convert_azimuth(&self.user_azimuth)
}
}
}
} else {
self.user_azimuth = [0., 0.];
self.azimuth = self.user_azimuth.clone();
}
}
}
fn set_radial(&mut self, radial: Option<Vec<f64>>, enable: Option<bool>) {
if let Some(enable) = enable {
if enable {
if let Some(radial) = radial {
if radial.len() < 2 {
self.radial = [0., 0.];
return;
}
self.radial[0] = radial[0].min(radial[1]);
self.radial[1] = radial[0].max(radial[1]);
}
} else {
self.radial = [0., 0.];
}
}
}
fn set_multi_column(&mut self, name: Option<String>) {
if let Some(name) = name {
self.multi_column = Arc::new(name);
}
}
fn set_azimuth_slices(&mut self, enable: Option<bool>) {
if let Some(enable) = enable {
self.azimuth_slices = enable;
}
}
fn set_cake(&mut self, enable: Option<bool>) {
if let Some(enable) = enable {
self.cake = enable;
}
}
fn set_azimuthal_integration(&mut self, enable: Option<bool>) {
if let Some(enable) = enable {
self.azimuthal = enable;
}
}
fn set_dark_background(&mut self, names: Option<Vec<String>>) -> Option<Vec<String>> {
let (array, errors) = names.parse_as_dark(&self.beamline, &self.detector, "background");
self.dark_bkg = array;
errors
}
fn set_dark_sample(&mut self, names: Option<Vec<String>>) -> Option<Vec<String>> {
let (array, errors) = names.parse_as_dark(&self.beamline, &self.detector, "sample");
self.dark_sample = array;
errors
}
fn set_save_edf(&mut self, enable: Option<bool>) {
if let Some(enable) = enable {
self.save_edf = enable;
}
}
fn set_flood(&mut self, flood: Option<String>) -> Option<String> {
if self.detector != Detector::Frelon {
self.flood = None;
return None;
}
if let Some(flood) = flood {
if flood.is_empty() {
self.flood = None;
return None;
}
let path = PathBuf::from(&flood);
match path.open(false) {
Ok(frame) => self.flood = Some(frame),
Err(e) => return Some(format!("Failed to open flood {}: {}", flood, e)),
}
}
None
}
fn set_background(
&mut self,
coefficient: Option<f64>,
names: Option<Vec<String>>,
) -> Option<Vec<String>> {
let coefficient = match coefficient {
Some(coefficient) => coefficient,
None => 1.0,
};
let names = match names {
Some(names) => {
if names.is_empty() {
self.bkg = None;
return None;
}
names
}
None => return None,
};
let mut errors = vec![];
let mut average_bkg = None;
for name in names.iter() {
let path = PathBuf::from(name);
info!("Averaging background file {:?}", path);
match path.open(false) {
Ok(mut frame) => {
let fp = self.normalize_bkg_frame(&mut frame, &mut errors);
match average_bkg {
None => average_bkg = Some(AverageFrame::new(&mut frame, fp)),
Some(ref mut af) => match af.append(&frame, &fp) {
Ok(_) => {}
Err(err) => {
warn!("Failed to average background file {:?}: {}", path, err)
}
},
}
}
Err(e) => errors.push(format!("{}: {}", name, e)),
}
}
if let Some(ref mut af) = average_bkg {
af.average(coefficient);
info!(
"Background: {} files have been averaged, transmission = {}, monitor = {}",
af.done, af.fp.transmission, af.fp.monitor,
);
}
self.bkg = average_bkg;
if errors.is_empty() {
None
} else {
Some(errors)
}
}
fn normalize_bkg_frame(
&self,
frame: &mut Box<dyn Frame>,
errors: &mut Vec<String>,
) -> FrameParams {
if let Some(dark) = self.dark_bkg.as_ref() {
let mut image = frame.array_mut();
if &dark.array == image {
image -= &dark.array;
} else {
errors.push("Dark and background dimensions are inconsistent".to_string());
}
}
let fp = self.beamline.monitor_and_transmission(frame);
if fp.monitor > 0. && self.norm == Normalization::Monitor {
let mut image = frame.array_mut();
image /= fp.monitor;
}
fp
}
fn set_radial_bins(&mut self, bins: Option<usize>) {
if let Some(bins) = bins {
if bins != self.r_bins {
self.i.set_radial_bins(bins);
self.r_bins = bins;
}
}
}
fn set_speed(&mut self, speed: Option<bool>) {
if let Some(speed) = speed {
self.speed = speed;
}
}
fn set_super_speed(&mut self, super_speed: Option<bool>) {
if let Some(super_speed) = super_speed {
self.super_speed = super_speed;
}
}
fn set_detector(&mut self, detector: Option<String>) -> Option<String> {
if let Some(detector) = detector {
self.detector = match detector.as_str() {
"" | "Pilatus" => Detector::Pilatus,
"Frelon" => Detector::Frelon,
_ => return Some(format!("Detector {} is not recognized", detector)),
}
}
None
}
fn set_extension(&mut self, extension: Option<String>) {
if let Some(mut extension) = extension {
if extension.is_empty() {
self.extension = Arc::new("dat".to_string());
} else {
if extension.starts_with('.') {
extension.remove(0);
}
self.extension = Arc::new(extension);
}
}
}
fn set_thickness(&mut self, thickness: Option<f64>) {
if let Some(thickness) = thickness {
self.thickness = thickness;
}
}
fn set_average(&mut self, average: Option<usize>) {
if let Some(average) = average {
self.average = average;
}
}
fn set_concentration(&mut self, concentration: Option<f64>) {
if let Some(concentration) = concentration {
self.concentration = concentration;
}
}
fn set_azimuthal_bins(&mut self, bins: Option<usize>) {
if let Some(bins) = bins {
if bins != self.a_bins {
self.i.set_azimuthal_bins(bins);
self.a_bins = bins;
}
}
}
fn set_beamline(&mut self, beamline: Option<String>) {
if let Some(beamline) = beamline {
self.beamline = match beamline.as_str() {
"Dubble" => Beamline::Dubble,
"SNBL" => Beamline::SNBL,
"None" => Beamline::None,
_ => return,
};
}
}
fn set_normalization(&mut self, norm: Option<String>, rng: Option<Vec<f64>>) -> Option<String> {
self.norm = match norm {
Some(normalization) => match normalization.as_ref() {
"Monitor" => Normalization::Monitor,
"Bkg" | "Background" => Normalization::Background,
"Median" => Normalization::Median,
"None" => Normalization::None,
_ => self.norm,
},
None => self.norm,
};
if let Some(range) = rng {
if range.len() < 2 {
return Some(format!(
"Not enough values in normalization range {:?}",
range
));
}
self.norm_range = [range[0], range[1]]
}
None
}
fn set_calibration(&mut self, factor: Option<f64>) {
if let Some(factor) = factor {
self.calibration = factor;
}
}
fn set_polarization(&mut self, pol_fac: Option<f64>) {
if let Some(pol_fac) = pol_fac {
self.i.set_polarization(pol_fac)
}
}
fn set_solid_angle(&mut self, sa: Option<bool>) {
if let Some(sa) = sa {
self.i.set_solid_angle(sa);
}
}
fn set_units(&mut self, units: Option<String>) {
if let Some(units) = units {
let units = match units.as_ref() {
"t" => Units::TwoTheta,
"q" => Units::Qnm,
"a" => Units::QA,
_ => Units::TwoTheta,
};
self.i.set_units(units);
}
}
fn set_running(&mut self, running: bool) {
self.running = running;
}
fn set_path(&mut self, path: Option<String>, subdir: Option<String>) -> Option<String> {
if let Some(path) = path {
if path.is_empty() {
return Some("Path cannot be empty".to_string());
}
let md = match metadata(&path) {
Ok(md) => md,
Err(e) => return Some(format!("Failed to get path metadata {}: {}", path, e)),
};
if !md.is_dir() {
return Some(format!("Path '{}' is not a directory", path));
}
self.path = path;
if let Some(subdir) = subdir {
self.subdir = Arc::new(PathBuf::from(subdir));
}
}
None
}
fn run(&mut self) {
self.set_running(true);
}
pub(crate) fn stop(&mut self) {
self.set_running(false);
}
pub fn is_running(&self) -> bool {
self.running
}
fn set_mask(&mut self, mask: Option<String>) -> Option<String> {
if let Some(mask) = mask {
if mask.is_empty() {
self.mask = None;
} else {
match mask.decompress() {
Ok(mask) => self.mask = Some(mask),
Err(_) => {
self.mask = None;
return Some(String::from("Opening masks by name not yet implemented"));
}
}
}
}
None
}
fn set_poni(&mut self, poni: Option<String>) -> Option<String> {
if let Some(poni) = poni {
if poni.is_empty() {
return Some(format!("Poni file must be specified"));
}
match poni.decompress() {
Ok(poni) => {
let poni: &[u8] = poni.as_ref();
match Poni::read_buffer(io::BufReader::new(poni)) {
Ok(poni) => self.i.set_poni(poni),
Err(e) => return Some(format!("Could not read poni: {}", e)),
}
}
Err(_) => match Poni::read_file(&poni) {
Ok(poni) => self.i.set_poni(poni),
Err(e) => return Some(format!("Could not read poni: {}", e)),
},
};
}
None
}
fn parse_spline(&mut self, spline: Option<String>) -> Option<String> {
if self.detector != Detector::Frelon {
self.spline = None;
return None;
}
if let Some(spline) = spline {
if spline.is_empty() {
self.spline = None;
return None;
}
match spline.decompress() {
Ok(spline) => {
let spline: &[u8] = spline.as_ref();
match Spline::parse(io::BufReader::new(spline)) {
Ok(spline) => self.set_spline(spline),
Err(e) => return Some(format!("Could not read spline: {}", e)),
}
}
Err(_) => match Spline::open(&spline) {
Ok(spline) => self.set_spline(spline),
Err(e) => return Some(format!("Could not read spline: {}", e)),
},
};
}
None
}
fn set_spline(&mut self, spline: Spline) {
let spline = Some(spline);
if self.spline != spline {
self.spline = spline;
self.d = Distortion::new();
}
}
pub fn speed(&self) -> bool {
self.speed
}
pub fn super_speed(&self) -> bool {
self.super_speed
}
fn is_correctable(&self) -> bool {
self.detector.is_frelon() && self.spline.is_some()
}
fn correct_source(&self, frame: &mut Box<dyn Frame>) -> Option<f64> {
let mut fp = self.beamline.monitor_and_transmission(frame);
let mut array = frame.array_mut();
if let Some(dark) = self.dark_sample.as_ref() {
if &dark.array == array {
array -= &dark.array
}
}
if let Some(flood) = self.flood.as_ref() {
let flood = flood.array();
if flood == array {
array /= flood;
}
}
if self.is_correctable() {
match self.d.correct(frame.array()) {
Ok(corrected) => frame.set_array(corrected),
Err(err) => {
error!("Error correcting spatial distortion: {}", err);
return None;
}
}
}
let mut array = frame.array_mut();
if self.norm == Normalization::Monitor && fp.monitor > 0. {
array /= fp.monitor;
}
if let Some(bkg) = self.bkg.as_ref() {
if &bkg.array == array {
if fp.transmission == 0. || self.thickness == 0. || bkg.fp.transmission == 0. {
array -= &bkg.array;
} else {
let t = fp.transmission / bkg.fp.transmission;
let m1 = (1. - self.concentration) * t;
let m2 = fp.transmission * self.thickness;
fp.transmission = t;
for (int, bkg) in array.data_mut().iter_mut().zip(bkg.array.data().iter()) {
*int = (*int - *bkg * m1) / m2;
}
}
}
}
if self.norm != Normalization::Background {
array *= self.calibration;
}
if let Some(mask) = self.mask.as_ref() {
if mask.len() == array.dim1() * array.dim2() / 8 + 1 {
for (j, val) in array.data_mut().iter_mut().enumerate() {
let mask_byte = unsafe { mask.get_unchecked(j / 8) };
if (mask_byte >> (j % 8) as u8) & 1 != 0 {
*val = -1.0;
}
}
}
}
Some(fp.transmission)
}
fn normalize_on_bkg(&self, p: &mut Pattern) {
if self.norm == Normalization::Background {
let mut sum = 0.;
for (pos, int) in p.positions.iter().zip(&p.intensity) {
if self.norm_range[0] == self.norm_range[1]
|| (*pos >= self.norm_range[0] && *pos < self.norm_range[1])
{
sum += *int;
}
}
if sum != 0. {
for (int, sigma) in p.intensity.iter_mut().zip(p.sigma.iter_mut()) {
*int *= self.calibration / sum;
*sigma = int.sqrt();
}
}
}
}
fn correct_results(&self, d: &mut Diffractogram) {
match &mut d.data {
PatternType::Azimuthal(ref mut pattern) => {
self.normalize_on_bkg(pattern);
let azimuth = [d.azimuth_min, d.azimuth_max];
match self.beamline {
Beamline::SNBL => pattern.azimuthal_snbl(&azimuth, &self.user_azimuth),
Beamline::Dubble => match self.itype {
IType::SAXS => pattern.azimuthal_dubble_saxs(&azimuth, &self.user_azimuth),
IType::WAXS => pattern.azimuthal_dubble_waxs(&azimuth, &self.user_azimuth),
},
Beamline::None => {}
}
}
PatternType::Radial(ref mut pattern) => {
self.normalize_on_bkg(pattern);
let radial = [d.radial_min, d.radial_max];
pattern.radial(&radial, &radial);
}
_ => {}
}
}
fn init(&mut self, frame: &mut Box<dyn Frame>) -> Vec<Box<Results>> {
debug!(
"Initializing integrator for {}x{}",
frame.dim1(),
frame.dim2()
);
self.i.init(frame.array());
if !self.d.is_initialized(frame.array()) {
if let Some(spline) = self.spline.as_mut() {
debug!(
"Initializing distortion for {}x{}",
frame.dim1(),
frame.dim2()
);
spline.calculate(frame.array());
self.d.init(frame.array(), &spline);
}
}
self.integrate(frame).unwrap()
}
fn integrate_one(&self, frame: &Box<dyn Frame>, it: IntegrationType) -> Option<Box<Results>> {
let data = Integrable {
array: frame.array(),
radial_range: &self.radial,
azimuthal_range: &self.azimuth,
integration_type: it,
};
match self.i.integrate(&data) {
Ok(d) => Some(Box::new(Results {
path: Default::default(),
name: Default::default(),
diffractogram: Some(d),
timestamp: py_stamp(),
transmission: 0.,
subdir: self.subdir.clone(),
ext: self.extension.clone(),
save_edf: self.save_edf,
azimuth: None,
id: Uuid::new_v4(),
})),
Err(err) => {
error!("Integration error: {}", err);
None
}
}
}
fn integrate_azimuthal_slices(&self, frame: &Box<dyn Frame>) -> Option<Vec<Box<Results>>> {
let mut patterns = vec![];
let mut range = [self.azimuth[0], self.azimuth[0] + self.azimuth[1]];
while range[0] < 360. {
let data = Integrable {
array: frame.array(),
radial_range: &self.radial,
azimuthal_range: &range,
integration_type: IntegrationType::Radial,
};
match self.i.integrate(&data) {
Ok(d) => {
patterns.push(Box::new(Results {
path: Default::default(),
name: Default::default(),
diffractogram: Some(d),
timestamp: py_stamp(),
transmission: 0.,
save_edf: self.save_edf,
ext: self.extension.clone(),
azimuth: Some(self.convert_azimuth(&range)),
subdir: self.subdir.clone(),
id: Uuid::new_v4(),
}));
range[0] += self.azimuth[1];
range[1] = range[0] + self.azimuth[1];
}
Err(_) => return None,
}
}
Some(patterns)
}
fn integrate(&self, frame: &mut Box<dyn Frame>) -> Option<Vec<Box<Results>>> {
let transmission = self.correct_source(frame)?;
let mut res = vec![];
if self.azimuth_slices && self.azimuth[1] > 0. {
res.append(&mut self.integrate_azimuthal_slices(frame)?);
} else {
res.push(self.integrate_one(frame, IntegrationType::Radial)?);
}
if self.azimuthal {
res.push(self.integrate_one(frame, IntegrationType::Azimuthal)?);
}
if self.cake {
res.push(self.integrate_one(frame, IntegrationType::Cake)?);
}
for r in &mut res {
self.correct_results(&mut r.diffractogram.as_mut().unwrap());
r.transmission = transmission;
}
Some(res)
}
}
trait FrameReader {
fn read(&self, beamline: &Beamline) -> Option<Box<dyn Frame>>;
}
trait SingleFrameReader: AsRef<Path> + Debug {
fn open(&self, skip: bool) -> BubbleResult<Box<dyn Frame>> {
match frame::open(self) {
Ok(frame) => {
let header = frame.header();
if !header.contains_key(KEY_DONT_TOUCH) {
return Ok(frame);
}
let key = match frame.get_header_i64(KEY_DONT_TOUCH) {
Ok(key) => key,
Err(_) => return Ok(frame),
};
if skip && key != 0 {
Err(BubbleError::AlreadyProcessed)
} else {
Ok(frame)
}
}
Err(e) => Err(BubbleError::FrameError(e)),
}
}
fn read(&self) -> Option<Box<dyn Frame>> {
match self.open(true) {
Ok(frame) => Some(frame),
Err(e) => {
debug!("Skipping {:?} due to error: {}", self, e);
None
}
}
}
}
impl SingleFrameReader for PathBuf {}
impl FrameReader for [PathBuf] {
fn read(&self, beamline: &Beamline) -> Option<Box<dyn Frame>> {
debug!(
"Averaging {}",
self.iter()
.map(|e| e.as_os_str().to_str().unwrap())
.join(" ")
);
let mut af = None;
for path in self {
info!("Averaging file {:?}", path);
if let Some(mut frame) = path.read() {
let fp = beamline.monitor_and_transmission(&frame);
match af {
None => af = Some(AverageFrame::new(&mut frame, fp)),
Some(ref mut af) => {
if let Err(err) = af.append(&frame, &fp) {
warn!("Failed to average file {:?}: {}", path, err)
}
}
}
}
}
match af {
None => None,
Some(mut af) => {
af.average(1.);
Some(Box::new(af))
}
}
}
}
#[derive(PartialEq, Message)]
#[rtype(result = "()")]
pub enum Task {
NotYet,
NotFrame(PathBuf),
One(PathBuf),
Many(Vec<PathBuf>),
}
impl Task {
fn is_not_yet(&self) -> bool {
self == &Task::NotYet
}
pub fn is_done(&self) -> bool {
!self.is_not_yet()
}
}
impl Handler<Task> for IntegrationActor {
type Result = ();
fn handle(&mut self, task: Task, _: &mut Self::Context) -> Self::Result {
let beamline = {
let i = self.i.read();
if !i.is_running() {
return;
} else {
i.beamline
}
};
let (frame, path) = match task {
Task::One(path) => (path.read(), path),
Task::Many(mut paths) => (paths.read(&beamline), paths.remove(paths.len() - 1)),
Task::NotYet => unreachable!("Task must never be sent being Task::NotYet!"),
Task::NotFrame(path) => {
info!("Failed to open as frame: {:?}", path);
self.results.do_send(TaskToDo(-1));
return;
}
};
info!("Processing {:?}", path);
match frame {
Some(frame) => self.parse_frame(frame, path),
None => {
info!("Failed to open as frame: {:?}", path);
self.results.do_send(TaskToDo(-1));
}
}
}
}
#[derive(PartialEq, Clone, Copy)]
enum Normalization {
None,
Monitor,
Background,
Median,
}
trait DarkParser {
fn parse_as_dark(
&self,
beamline: &Beamline,
detector: &Detector,
msg: &str,
) -> (Option<AverageFrame>, Option<Vec<String>>);
}
impl DarkParser for Option<Vec<String>> {
fn parse_as_dark(
&self,
beamline: &Beamline,
detector: &Detector,
msg: &str,
) -> (Option<AverageFrame>, Option<Vec<String>>) {
let mut errors = vec![];
let mut af = None;
match detector {
Detector::Frelon => {
let names = match self {
Some(names) => {
if names.is_empty() {
return (None, None);
}
names
}
None => return (None, None),
};
for name in names {
let path = PathBuf::from(name);
info!("Averaging dark file {:?}", path);
match path.open(false) {
Ok(mut frame) => {
let fp = beamline.monitor_and_transmission(&frame);
match af {
None => af = Some(AverageFrame::new(&mut frame, fp)),
Some(ref mut af) => match af.append(&frame, &fp) {
Ok(_) => {}
Err(err) => {
warn!("Failed to average dark files {:?}: {}", path, err)
}
},
}
}
Err(e) => errors.push(format!("{}: {}", name, e)),
}
}
if let Some(ref mut af) = af {
af.average(1.);
info!("Dark {}: {} files have been averaged", msg, af.done);
}
}
_ => {}
}
if errors.is_empty() {
(af, None)
} else {
(af, Some(errors))
}
}
}
trait PostProcessor {
fn recalculate(&mut self, range: &[f64; 2], user_range: &[f64; 2], inverted: bool);
fn radial(&mut self, range: &[f64; 2], user_range: &[f64; 2]);
fn azimuthal_snbl(&mut self, range: &[f64; 2], user_range: &[f64; 2]);
fn azimuthal_dubble_saxs(&mut self, range: &[f64; 2], user_range: &[f64; 2]);
fn azimuthal_dubble_waxs(&mut self, range: &[f64; 2], user_range: &[f64; 2]);
}
impl PostProcessor for Pattern {
fn recalculate(&mut self, range: &[f64; 2], user_range: &[f64; 2], inverted: bool) {
if range[0] == range[1] {
return;
}
let last = self.positions.len() - 1;
let mut min = 0;
let mut max = last;
for (i, p) in self.positions.iter().enumerate() {
if min == 0 && range[0] <= *p {
min = i;
}
if max == last && range[1] <= *p {
max = i;
}
if min != 0 && max != last {
break;
}
}
if inverted {
min += 1;
} else if max > 2 {
max -= 1;
}
let mut positions = self.positions.as_ref().clone();
positions.drain_garbage(min, max);
if *range != *user_range {
let mut pos = user_range[0];
let step = (user_range[1] - pos) / positions.len() as f64;
for val in &mut positions {
*val = pos;
pos += step;
}
}
self.intensity.drain_garbage(min, max);
self.sigma.drain_garbage(min, max);
self.positions = Arc::new(positions);
}
fn radial(&mut self, range: &[f64; 2], user_range: &[f64; 2]) {
self.recalculate(range, user_range, false);
}
fn azimuthal_snbl(&mut self, range: &[f64; 2], user_range: &[f64; 2]) {
self.recalculate(range, user_range, true);
self.intensity.reverse();
self.sigma.reverse();
}
fn azimuthal_dubble_saxs(&mut self, range: &[f64; 2], user_range: &[f64; 2]) {
self.recalculate(range, user_range, false);
}
fn azimuthal_dubble_waxs(&mut self, range: &[f64; 2], user_range: &[f64; 2]) {
self.recalculate(range, user_range, false);
let k = self.positions.len() / 2;
self.intensity.rotate_right(k);
self.sigma.rotate_right(k);
}
}
trait GarbageDrainer<T> {
fn drain_garbage(&mut self, min: usize, max: usize);
}
impl<T> GarbageDrainer<T> for Vec<T> {
fn drain_garbage(&mut self, min: usize, max: usize) {
self.rotate_left(min);
unsafe { self.set_len(max - min) };
}
}
pub type ResultID = Uuid;
#[derive(Message)]
#[rtype(result = "()")]
pub struct Results {
pub path: Arc<PathBuf>,
pub name: PathBuf,
diffractogram: Option<Diffractogram>,
pub timestamp: f64,
pub transmission: f64,
subdir: Arc<PathBuf>,
ext: Arc<String>,
save_edf: bool,
azimuth: Option<[f64; 2]>,
pub id: ResultID,
}
impl Results {
pub fn empty() -> Results {
Results {
path: Arc::new(Default::default()),
name: Default::default(),
diffractogram: None,
timestamp: 0.0,
transmission: 0.0,
subdir: Arc::new(Default::default()),
ext: Arc::new(String::new()),
save_edf: false,
azimuth: None,
id: Uuid::new_v4(),
}
}
fn parse_name(&self, frame_number: Option<usize>) -> String {
let fs = self.path.file_stem().unwrap().to_str().unwrap();
let frame_number = match frame_number {
None => String::new(),
Some(frame_number) => format!("_{:05}", frame_number),
};
match self.diffractogram.as_ref().unwrap().data {
PatternType::Radial(_) => {
if let Some(az) = self.azimuth.as_ref() {
format!(
"{}_azimuthal_slice_{}_{}{}.{}",
fs, az[0], az[1], frame_number, self.ext
)
} else {
format!("{}{}.{}", fs, frame_number, self.ext)
}
}
PatternType::Azimuthal(_) => format!("{}_azimuthal{}.{}", fs, frame_number, self.ext),
PatternType::Cake(_) => format!("{}_cake{}.edf", fs, frame_number),
PatternType::None => unimplemented!("PatternType::None is not implemented"),
}
}
fn make_filename(&mut self, frame_number: Option<usize>) {
self.name = self.path.parent().unwrap().to_path_buf();
self.name.push(self.subdir.as_path());
self.name.push(self.parse_name(frame_number));
}
fn save_normalized_image(&self) -> io::Result<()> {
let mut header = Header::new();
header.insert(KEY_DONT_TOUCH.to_string(), HeaderEntry::Number(1));
let mut writer = BufWriter::new(File::create(&self.name)?);
Edf::save_array(
&self.diffractogram.as_ref().unwrap().image,
&mut header,
&mut writer,
DataType::F64,
)?;
self.name.reset_permission(false)?;
Ok(())
}
fn save(&mut self, frame_number: Option<usize>) -> io::Result<()> {
self.make_filename(frame_number);
match &self.diffractogram.as_ref().unwrap().data {
PatternType::Radial(p) | PatternType::Azimuthal(p) => {
let dir = self.name.parent().unwrap();
fs::create_dir_all(dir)?;
dir.reset_permission(true)?;
let mut writer = BufWriter::new(fs::File::create(&self.name)?);
if self.transmission != 0. {
writeln!(writer, "# Transmission coefficient: {}", self.transmission)?;
}
p.to_text(&mut writer)?;
self.name.reset_permission(false)?;
if self.save_edf {
self.name.set_extension("edf");
self.save_normalized_image()?;
self.name.set_extension(&self.ext.as_str());
}
}
PatternType::Cake(c) => {
let mut h = Header::new();
h.insert(KEY_DONT_TOUCH.to_owned(), HeaderEntry::Number(1));
h.insert(
KEY_BUBBLE_CAKE.to_owned(),
HeaderEntry::String(format!(
"{} {} {} {}",
c.radial_first(),
c.radial_last(),
c.azimuthal_first(),
c.azimuthal_last(),
)),
);
let mut writer = BufWriter::new(File::create(&self.name)?);
Edf::save_array(&c.cake, &mut h, &mut writer, DataType::F64)?;
self.name.reset_permission(false)?;
}
PatternType::None => {}
}
Ok(())
}
pub fn pack(&self, bin: usize, speed: bool) -> Packed {
let mut packed = Packed::new();
let p = vec![];
let mut buf = Cursor::new(p);
if let Some(d) = &self.diffractogram {
match &d.data {
PatternType::Radial(p) | PatternType::Azimuthal(p) => {
if let Ok(_) = p.to_text(&mut buf) {
packed.pattern_size = buf.get_ref().len();
packed.pattern = buf.get_ref().compress()
}
}
_ => {}
}
}
if speed {
return packed;
}
if let Some(p) = &self.diffractogram {
let mut header = Header::new();
header.insert(KEY_DONT_TOUCH.to_owned(), HeaderEntry::Number(1));
let image = &p.image;
let p = Vec::with_capacity(image.len() * mem::size_of::<f32>());
let mut buf = Cursor::new(p);
let resized = image.resize(bin);
if let Ok(_) = if let Some(resized) = resized {
Edf::save_array(&resized, &mut header, &mut buf, edf::DataType::F32)
} else {
Edf::save_array(image, &mut header, &mut buf, edf::DataType::F32)
} {
packed.frame_size = buf.get_ref().len();
packed.frame = buf.get_ref().compress();
}
}
packed
}
}
impl Beamline {
fn monitor_and_transmission(&self, frame: &Box<dyn Frame>) -> FrameParams {
let params = match self {
Beamline::SNBL => (frame.get_header_float(cbf::KEY_FLUX), 0., 0.),
Beamline::Dubble => {
let mut monitor = frame.get_header_float(KEY_DUBBLE_MONITOR);
if monitor == 0. {
monitor = frame.sum();
}
if monitor == 0. {
(0., 0., 0.)
} else {
let photo = frame.get_header_float(KEY_DUBBLE_PHOTO);
(monitor, photo / monitor, photo)
}
}
Beamline::None => (0., 0., 0.),
};
FrameParams {
monitor: params.0,
transmission: params.1,
photo: params.2,
}
}
}
trait FilePermission {
fn reset_permission(&self, is_dir: bool) -> io::Result<()>;
}
impl<T: AsRef<Path>> FilePermission for T {
#[cfg(not(target_os = "windows"))]
fn reset_permission(&self, is_dir: bool) -> io::Result<()> {
if is_dir {
fs::set_permissions(&self, fs::Permissions::from_mode(0o755))
} else {
fs::set_permissions(&self, fs::Permissions::from_mode(0o644))
}
}
#[cfg(target_os = "windows")]
fn reset_permission(&self, _is_dir: bool) -> io::Result<()> {
Ok(())
}
}
pub struct Packed {
pub frame: String,
pub frame_size: usize,
pub pattern: String,
pub pattern_size: usize,
}
impl Packed {
pub fn new() -> Packed {
Packed {
frame: String::new(),
frame_size: 0,
pattern: String::new(),
pattern_size: 0,
}
}
}