1use std::{ io::*, collections::BTreeSet, collections::BTreeMap };
5use std::sync::{ Arc, atomic::AtomicBool };
6use byteorder::{ ReadBytesExt, BigEndian };
7use mp4parse::{ MediaContext, TrackType };
8use memchr::memmem;
9
10use crate::tags_impl::*;
11
12pub fn to_hex(data: &[u8]) -> String {
13 let mut ret = String::with_capacity(data.len() * 3);
14 for b in data {
15 ret.push_str(&format!("{:02x} ", b));
16 }
17 ret
18}
19
20#[derive(Debug, Clone, Default)]
21pub struct SampleInfo {
22 pub sample_index: u64,
23 pub track_index: usize,
24 pub timestamp_ms: f64,
25 pub duration_ms: f64,
26 pub tag_map: Option<GroupedTagMap>
27}
28
29pub fn verify_and_fix_mp4_structure(bytes: &mut Vec<u8>) {
32 crate::try_block!({
33 let mut good_size = 0;
34 let mut pos = 0;
35 while pos < bytes.len() - 1 {
36 let start_pos = pos;
37 let mut len = (&bytes[pos..]).read_u32::<BigEndian>().ok()? as u64;
38 pos += 4;
39 let name_good = bytes.len() >= pos + 4 && bytes[pos..pos+4].iter().all(|x| x.is_ascii() && *x > 13);
40 pos += 4;
41 if len == 1 { len = (&bytes[pos..]).read_u64::<BigEndian>().ok()?;
43 }
44 pos = start_pos + len as usize;
45 let size_good = bytes.len() >= pos;
46 if name_good && size_good {
47 good_size = pos;
48 } else {
49 break;
50 }
51 }
52 if bytes.len() > good_size {
53 log::warn!("Garbage found at the end of the file, removing {} bytes from the end.", bytes.len() - good_size);
54 bytes.resize(good_size, 0);
55 }
56 });
57}
58
59pub fn hide_wave_box(all: &mut Vec<u8>) {
61 let mut offs = 0;
62 while let Some(pos) = memchr::memmem::find(&all[offs..], b"wave") {
63 if all.len() > offs+pos+12 && &all[offs+pos+8..offs+pos+12] == b"frma" {
64 all[offs + pos + 3] = b'_';
65 return;
66 }
67 offs += pos + 4;
68 }
69}
70
71pub fn parse_mp4<T: Read + Seek>(stream: &mut T, size: usize) -> mp4parse::Result<mp4parse::MediaContext> {
72 if size > 10*1024*1024 {
73 let mut all = read_beginning_and_end(stream, size, 2*1024*1024)?;
77 if let Some(pos) = memchr::memmem::find(&all, b"mdat") {
78 let how_much_less = (size - all.len()) as u64;
79 let mut len = (&all[pos-4..]).read_u32::<BigEndian>()? as u64;
80 if len == 1 { len = (&all[pos+4..]).read_u64::<BigEndian>()? - how_much_less;
82 all[pos+4..pos+12].copy_from_slice(&len.to_be_bytes());
83 } else {
84 len -= how_much_less;
85 all[pos-4..pos].copy_from_slice(&(len as u32).to_be_bytes());
86 }
87
88 verify_and_fix_mp4_structure(&mut all);
89 hide_wave_box(&mut all);
90
91 let mut c = std::io::Cursor::new(&all);
92 return mp4parse::read_mp4(&mut c);
93 }
94 }
95 mp4parse::read_mp4(stream)
96}
97
98pub fn get_track_samples<F, T: Read + Seek>(stream: &mut T, size: usize, typ: mp4parse::TrackType, single: bool, max_sample_size: Option<usize>, mut callback: F, cancel_flag: Arc<AtomicBool>) -> Result<MediaContext>
99 where F: FnMut(SampleInfo, &[u8], u64)
100{
101
102 let ctx = parse_mp4(stream, size).or_else(|_| mp4parse::read_mp4(stream)).unwrap();
103
104 let mut track_index = 0;
105 for x in &ctx.tracks {
109 if x.track_type == typ {
110 if let Some(samples) = mp4parse::unstable::create_sample_table(&x, 0.into()) {
117 let mut sample_data = Vec::new();
118 let mut sample_index = 0u64;
119 for x in samples {
120 if cancel_flag.load(std::sync::atomic::Ordering::Relaxed) { break; }
121
122 let mut sample_size = (x.end_offset.0 - x.start_offset.0) as usize;
123 if let Some(max_sample_size) = max_sample_size {
124 if sample_size > max_sample_size {
125 sample_size = max_sample_size;
126 }
127 }
128 let sample_timestamp_ms = x.start_composition.0 as f64 / 1000.0;
129 let sample_duration_ms = (x.end_composition.0 - x.start_composition.0) as f64 / 1000.0;
130 if sample_size > 4 {
131 if sample_data.len() != sample_size {
132 sample_data.resize(sample_size, 0u8);
133 }
134
135 stream.seek(SeekFrom::Start(x.start_offset.0 as u64))?;
136 stream.read_exact(&mut sample_data[..])?;
137
138 callback(SampleInfo { sample_index, track_index, timestamp_ms: sample_timestamp_ms, duration_ms: sample_duration_ms, tag_map: None }, &sample_data, x.start_offset.0 as u64);
139
140 sample_index += 1;
142 }
143 }
144 if single {
145 break;
146 }
147 }
148 }
150 track_index += 1;
151 }
152 Ok(ctx)
153}
154
155pub fn get_metadata_track_samples<F, T: Read + Seek>(stream: &mut T, size: usize, single: bool, callback: F, cancel_flag: Arc<AtomicBool>) -> Result<MediaContext>
156 where F: FnMut(SampleInfo, &[u8], u64)
157{
158 get_track_samples(stream, size, mp4parse::TrackType::Metadata, single, None, callback, cancel_flag)
159}
160pub fn get_other_track_samples<F, T: Read + Seek>(stream: &mut T, size: usize, single: bool, callback: F, cancel_flag: Arc<AtomicBool>) -> Result<MediaContext>
161 where F: FnMut(SampleInfo, &[u8], u64)
162{
163 get_track_samples(stream, size, mp4parse::TrackType::Unknown, single, None, callback, cancel_flag)
164}
165
166pub fn read_beginning_and_end<T: Read + Seek>(stream: &mut T, stream_size: usize, read_size: usize) -> Result<Vec<u8>> {
167 let mut all = vec![0u8; read_size*2];
168
169 stream.seek(SeekFrom::Start(0))?;
170
171 if stream_size > read_size * 2 {
172 let read1 = stream.read(&mut all[..read_size])?;
173
174 stream.seek(SeekFrom::End(-(read_size as i64)))?;
175 let read2 = stream.read(&mut all[read1..])?;
176
177 all.resize(read1+read2, 0);
178 } else {
179 let read = stream.read(&mut all)?;
180 all.resize(read, 0);
181 }
182
183 stream.seek(SeekFrom::Start(0))?;
184
185 Ok(all)
186}
187
188#[derive(Default, serde::Serialize, serde::Deserialize, Clone, Debug)]
189pub struct IMUData {
190 pub timestamp_ms: f64,
191 pub gyro: Option<[f64; 3]>,
192 pub accl: Option<[f64; 3]>,
193 pub magn: Option<[f64; 3]>
194}
195
196
197pub fn normalized_imu(input: &crate::Input, orientation: Option<String>) -> Result<Vec<IMUData>> {
198 let mut timestamp = 0f64;
199 let mut first_timestamp = None;
200
201 let mut final_data = Vec::<IMUData>::with_capacity(10000);
202 let mut data_index = 0;
203
204 let mut fix_timestamps = false;
205
206 if let Some(ref samples) = input.samples {
207 for info in samples {
208 if info.tag_map.is_none() { continue; }
209
210 let grouped_tag_map = info.tag_map.as_ref().unwrap();
211
212 let first_frame_ts = crate::try_block!(f64, {
214 (grouped_tag_map.get(&GroupId::Default)?.get_t(TagId::Metadata) as Option<&serde_json::Value>)?
215 .as_object()?
216 .get("first_frame_timestamp")?
217 .as_i64()? as f64 / 1000.0
218 }).unwrap_or_default();
219 let is_insta360_raw_gyro = crate::try_block!(bool, {
220 (grouped_tag_map.get(&GroupId::Default)?.get_t(TagId::Metadata) as Option<&serde_json::Value>)?
221 .as_object()?
222 .get("is_raw_gyro")?
223 .as_bool()?
224 }).unwrap_or_default();
225
226 for (group, map) in grouped_tag_map {
227 if group == &GroupId::Gyroscope || group == &GroupId::Accelerometer || group == &GroupId::Magnetometer {
228 let raw2unit = crate::try_block!(f64, {
229 match &map.get(&TagId::Scale)?.value {
230 TagValue::i16(v) => *v.get() as f64,
231 TagValue::f32(v) => *v.get() as f64,
232 TagValue::f64(v) => *v.get(),
233 _ => 1.0
234 }
235 }).unwrap_or(1.0);
236
237 let unit2deg = crate::try_block!(f64, {
238 match (map.get_t(TagId::Unit) as Option<&String>)?.as_str() {
239 "rad/s" => 180.0 / std::f64::consts::PI, "g" => 9.80665, _ => 1.0
242 }
243 }).unwrap_or(1.0);
244
245 let mut io = match map.get_t(TagId::Orientation) as Option<&String> {
246 Some(v) => v.clone(),
247 None => "XYZ".into()
248 };
249 io = input.normalize_imu_orientation(io);
250 if let Some(imuo) = &orientation {
251 io = imuo.clone();
252 }
253 let io = io.as_bytes();
254
255 if let Some(taginfo) = map.get(&TagId::Data) {
256 match &taginfo.value {
257 TagValue::Vec_Vector3_i16(arr) => {
259 let arr = arr.get();
260 let reading_duration = info.duration_ms / arr.len() as f64;
261 fix_timestamps = true;
262
263 for (j, v) in arr.iter().enumerate() {
264 if final_data.len() <= data_index + j {
265 final_data.resize_with(data_index + j + 1, Default::default);
266 final_data[data_index + j].timestamp_ms = timestamp;
267 timestamp += reading_duration;
268 }
269 let itm = v.clone().into_scaled(&raw2unit, &unit2deg).orient(io);
270 if group == &GroupId::Gyroscope { final_data[data_index + j].gyro = Some([ itm.x, itm.y, itm.z ]); }
271 else if group == &GroupId::Accelerometer { final_data[data_index + j].accl = Some([ itm.x, itm.y, itm.z ]); }
272 else if group == &GroupId::Magnetometer { final_data[data_index + j].magn = Some([ itm.x, itm.y, itm.z ]); }
273 }
274 },
275 TagValue::Vec_TimeVector3_f64(arr) => {
277 for (j, v) in arr.get().iter().enumerate() {
278 if v.t < first_frame_ts { continue; } if final_data.len() <= data_index + j {
280 final_data.resize_with(data_index + j + 1, Default::default);
281 let timestamp_multiplier = if is_insta360_raw_gyro { 1.0 } else { 1000.0 };
282 final_data[data_index + j].timestamp_ms = (v.t - first_frame_ts) * timestamp_multiplier;
283 if first_timestamp.is_none() {
284 first_timestamp = Some(final_data[data_index + j].timestamp_ms);
285 final_data[data_index + j].timestamp_ms = 0.0;
286 } else {
287 final_data[data_index + j].timestamp_ms -= first_timestamp.unwrap();
288 }
289 }
290 let itm = v.clone().into_scaled(&raw2unit, &unit2deg).orient(io);
291 if group == &GroupId::Gyroscope { final_data[data_index + j].gyro = Some([ itm.x, itm.y, itm.z ]); }
292 else if group == &GroupId::Accelerometer { final_data[data_index + j].accl = Some([ itm.x, itm.y, itm.z ]); }
293 else if group == &GroupId::Magnetometer { final_data[data_index + j].magn = Some([ itm.x, itm.y, itm.z ]); }
294 }
295 },
296 _ => ()
297 }
298 }
299 }
300 }
301 data_index = final_data.len();
302 }
303 }
304
305 if fix_timestamps && !final_data.is_empty() {
306 let avg_diff = {
307 if input.camera_type() == "GoPro" {
308 crate::gopro::GoPro::get_avg_sample_duration(input.samples.as_ref().unwrap(), &GroupId::Gyroscope)
309 } else {
310 let mut total_duration_ms = 0.0;
311 for info in input.samples.as_ref().unwrap() {
312 total_duration_ms += info.duration_ms;
313 }
314 Some(total_duration_ms / final_data.len() as f64)
315 }
316 };
317 if let Some(avg_diff) = avg_diff {
318 if avg_diff > 0.0 {
319 for (i, x) in final_data.iter_mut().enumerate() {
320 x.timestamp_ms = avg_diff * i as f64;
321 }
322 }
323 }
324 }
325
326 Ok(final_data)
327}
328
329pub fn normalized_imu_interpolated(input: &crate::Input, orientation: Option<String>) -> Result<Vec<IMUData>> {
330 let mut first_timestamp = None;
331
332 let mut timestamp = (0.0, 0.0, 0.0);
333
334 let mut gyro_map = BTreeMap::new();
335 let mut accl_map = BTreeMap::new();
336 let mut magn_map = BTreeMap::new();
337
338 let mut all_timestamps = BTreeSet::new();
339
340 if let Some(ref samples) = input.samples {
341 let mut reading_duration =
342 if input.camera_type() == "GoPro" {
343 (
344 crate::gopro::GoPro::get_avg_sample_duration(samples, &GroupId::Gyroscope),
345 crate::gopro::GoPro::get_avg_sample_duration(samples, &GroupId::Accelerometer),
346 crate::gopro::GoPro::get_avg_sample_duration(samples, &GroupId::Magnetometer),
347 )
348 } else {
349 let mut total_len = (0, 0, 0);
350 for grouped_tag_map in samples.iter().filter_map(|v| v.tag_map.as_ref()) {
351 for (group, map) in grouped_tag_map {
352 if let Some(taginfo) = map.get(&TagId::Data) {
353 if let TagValue::Vec_Vector3_i16(arr) = &taginfo.value {
354 match group {
355 GroupId::Gyroscope => total_len.0 += arr.get().len(),
356 GroupId::Accelerometer => total_len.1 += arr.get().len(),
357 GroupId::Magnetometer => total_len.2 += arr.get().len(),
358 _ => {}
359 }
360 }
361 }
362 }
363 }
364
365 let mut total_duration_ms = 0.0;
366 for info in samples {
367 total_duration_ms += info.duration_ms;
368 }
369 (
370 if total_len.0 > 0 { Some(total_duration_ms / total_len.0 as f64) } else { None },
371 if total_len.1 > 0 { Some(total_duration_ms / total_len.1 as f64) } else { None },
372 if total_len.2 > 0 { Some(total_duration_ms / total_len.2 as f64) } else { None }
373 )
374 };
375 log::debug!("Reading duration: {:?}", reading_duration);
376 if let Some(grd) = reading_duration.0 {
377 if let Some(ard) = reading_duration.1 {
378 if (grd - ard).abs() < 0.1 {
379 reading_duration.0 = Some(grd.max(ard));
380 reading_duration.1 = Some(grd.max(ard));
381 }
382 }
383 if let Some(mrd) = reading_duration.2 {
384 if (grd - mrd).abs() < 0.1 {
385 reading_duration.0 = Some(grd.max(mrd));
386 reading_duration.2 = Some(grd.max(mrd));
387 }
388 }
389 }
390
391 for info in samples {
392 if info.tag_map.is_none() { continue; }
393
394 let grouped_tag_map = info.tag_map.as_ref().unwrap();
395
396 let first_frame_ts = crate::try_block!(f64, {
398 (grouped_tag_map.get(&GroupId::Default)?.get_t(TagId::Metadata) as Option<&serde_json::Value>)?
399 .as_object()?
400 .get("first_frame_timestamp")?
401 .as_i64()? as f64 / 1000.0
402 }).unwrap_or_default();
403 let is_insta360_raw_gyro = crate::try_block!(bool, {
404 (grouped_tag_map.get(&GroupId::Default)?.get_t(TagId::Metadata) as Option<&serde_json::Value>)?
405 .as_object()?
406 .get("is_raw_gyro")?
407 .as_bool()?
408 }).unwrap_or_default();
409 let timestamp_multiplier = if is_insta360_raw_gyro { 1.0 } else { 1000.0 };
410
411 for (group, map) in grouped_tag_map {
412 if group == &GroupId::Gyroscope || group == &GroupId::Accelerometer || group == &GroupId::Magnetometer {
413 let raw2unit = crate::try_block!(f64, {
414 match &map.get(&TagId::Scale)?.value {
415 TagValue::i16(v) => *v.get() as f64,
416 TagValue::f32(v) => *v.get() as f64,
417 TagValue::f64(v) => *v.get(),
418 _ => 1.0
419 }
420 }).unwrap_or(1.0);
421
422 let unit2deg = crate::try_block!(f64, {
423 match (map.get_t(TagId::Unit) as Option<&String>)?.as_str() {
424 "rad/s" => 180.0 / std::f64::consts::PI, "g" => 9.80665, _ => 1.0
427 }
428 }).unwrap_or(1.0);
429
430 let mut io = match map.get_t(TagId::Orientation) as Option<&String> {
431 Some(v) => v.clone(),
432 None => "XYZ".into()
433 };
434 io = input.normalize_imu_orientation(io);
435 if let Some(imuo) = &orientation {
436 io = imuo.clone();
437 }
438 let io = io.as_bytes();
439
440 if let Some(taginfo) = map.get(&TagId::Data) {
441 match &taginfo.value {
442 TagValue::Vec_Vector3_i16(arr) => {
444 let arr = arr.get();
445
446 for v in arr {
447 let itm = v.clone().into_scaled(&raw2unit, &unit2deg).orient(io);
448 if group == &GroupId::Gyroscope { let ts = (timestamp.0 * 1000.0f64).round() as i64; gyro_map.insert(ts, itm); timestamp.0 += reading_duration.0.unwrap(); all_timestamps.insert(ts); }
449 else if group == &GroupId::Accelerometer { let ts = (timestamp.1 * 1000.0f64).round() as i64; accl_map.insert(ts, itm); timestamp.1 += reading_duration.1.unwrap(); all_timestamps.insert(ts); }
450 else if group == &GroupId::Magnetometer { let ts = (timestamp.2 * 1000.0f64).round() as i64; magn_map.insert(ts, itm); timestamp.2 += reading_duration.2.unwrap(); all_timestamps.insert(ts); }
451 }
452 },
453 TagValue::Vec_TimeVector3_f64(arr) => {
454 for v in arr.get() {
455 if v.t < first_frame_ts { continue; } let mut timestamp_ms = (v.t - first_frame_ts) * timestamp_multiplier;
458 if first_timestamp.is_none() {
459 first_timestamp = Some(timestamp_ms);
460 }
461 timestamp_ms -= first_timestamp.unwrap();
462
463 let timestamp_us = (timestamp_ms * 1000.0).round() as i64;
464 all_timestamps.insert(timestamp_us);
465
466 let itm = v.clone().into_scaled(&raw2unit, &unit2deg).orient(io);
467 if group == &GroupId::Gyroscope { gyro_map.insert(timestamp_us, itm); }
468 else if group == &GroupId::Accelerometer { accl_map.insert(timestamp_us, itm); }
469 else if group == &GroupId::Magnetometer { magn_map.insert(timestamp_us, itm); }
470 }
471 },
472 _ => ()
473 }
474 }
475 }
476 }
477 }
478 }
479
480 fn get_at_timestamp(ts: i64, map: &BTreeMap<i64, Vector3<f64>>) -> Option<[f64; 3]> {
481 if map.is_empty() { return None; }
482 if let Some(v) = map.get(&ts) { return Some([v.x, v.y, v.z]); }
483
484 if let Some((k1, v1)) = map.range(..=ts).next_back() {
485 if let Some((k2, v2)) = map.range(ts..).next() {
486 let time_delta = (k2 - k1) as f64;
487 let fract = (ts - k1) as f64 / time_delta;
488 return Some([
490 v1.x * (1.0 - fract) + (v2.x * fract),
491 v1.y * (1.0 - fract) + (v2.y * fract),
492 v1.z * (1.0 - fract) + (v2.z * fract),
493 ]);
494 }
495 }
496 None
497 }
498
499 let mut final_data = Vec::with_capacity(gyro_map.len());
500 for x in &all_timestamps {
501 final_data.push(IMUData {
502 timestamp_ms: *x as f64 / 1000.0,
503 gyro: get_at_timestamp(*x, &gyro_map),
504 accl: get_at_timestamp(*x, &accl_map),
505 magn: get_at_timestamp(*x, &magn_map)
506 });
507 }
508
509 Ok(final_data)
510}
511
512pub fn multiply_quats(p: (f64, f64, f64, f64), q: (f64, f64, f64, f64)) -> Quaternion<f64> {
513 Quaternion {
514 w: p.0*q.0 - p.1*q.1 - p.2*q.2 - p.3*q.3,
515 x: p.0*q.1 + p.1*q.0 + p.2*q.3 - p.3*q.2,
516 y: p.0*q.2 - p.1*q.3 + p.2*q.0 + p.3*q.1,
517 z: p.0*q.3 + p.1*q.2 - p.2*q.1 + p.3*q.0
518 }
519}
520
521pub fn find_between_with_offset(buffer: &[u8], from: &[u8], to: u8, offset: i32) -> Option<String> {
522 let pos = memmem::find(buffer, from)?;
523 let end = memchr::memchr(to, &buffer[pos+from.len()..])?;
524 Some(String::from_utf8_lossy(&buffer[(pos as i32 + from.len() as i32 + offset) as usize..pos+from.len()+end]).into())
525}
526
527pub fn find_between(buffer: &[u8], from: &[u8], to: u8) -> Option<String> {
528 find_between_with_offset(buffer, from, to, 0)
529}
530
531pub fn insert_tag(map: &mut GroupedTagMap, tag: TagDescription) {
532 let group_map = map.entry(tag.group.clone()).or_insert_with(TagMap::new);
533 group_map.insert(tag.id.clone(), tag);
534}
535
536pub fn create_csv_map<'a, 'b>(row: &'b csv::StringRecord, headers: &'a Vec<String>) -> BTreeMap<&'a str, &'b str> {
537 headers.iter().zip(row).map(|(a, b)| (&a[..], b.trim())).collect()
538}
539pub fn create_csv_map_hdr<'a, 'b>(row: &'b csv::StringRecord, headers: &'a csv::StringRecord) -> BTreeMap<&'a str, &'b str> {
540 headers.iter().zip(row).map(|(a, b)| (a, b)).collect()
541}
542
543pub fn get_fps_from_track(track: &mp4parse::Track) -> Option<f64> {
544 if let Some(ref stts) = track.stts {
545 if !stts.samples.is_empty() {
546 let samples: u32 = stts.samples.iter().map(|v| v.sample_count).sum();
547 let timescale = track.timescale?;
548 let duration = track.duration?;
549 let duration_us = duration.0 as f64 * 1000_000.0 / timescale.0 as f64;
550 let us_per_frame = duration_us / samples as f64;
551 return Some(1000_000.0 / us_per_frame);
552 }
553 }
554 None
555}
556pub fn get_video_metadata<T: Read + Seek>(stream: &mut T, filesize: usize) -> Result<(usize, usize, f64, f64)> { let mp = parse_mp4(stream, filesize)?;
558 for track in mp.tracks {
559 if track.track_type == TrackType::Video {
560 let mut duration_sec = 0.0;
561 if let Some(d) = track.duration {
562 if let Some(ts) = track.timescale {
563 duration_sec = d.0 as f64 / ts.0 as f64;
564 }
565 }
566 if let Some(ref tkhd) = track.tkhd {
567 let w = tkhd.width >> 16;
568 let h = tkhd.height >> 16;
569 let matrix = (
570 tkhd.matrix.a >> 16,
571 tkhd.matrix.b >> 16,
572 tkhd.matrix.c >> 16,
573 tkhd.matrix.d >> 16,
574 );
575 let _rotation = match matrix {
576 (0, 1, -1, 0) => 90, (-1, 0, 0, -1) => 180, (0, -1, 1, 0) => 270, _ => 0,
580 };
581 let fps = get_fps_from_track(&track).unwrap_or_default();
582 return Ok((w as usize, h as usize, fps, duration_sec));
583 }
584 }
585 }
586 Err(ErrorKind::Other.into())
587}
588
589#[macro_export]
590macro_rules! try_block {
591 ($type:ty, $body:block) => {
592 (|| -> Option<$type> {
593 Some($body)
594 }())
595 };
596 ($body:block) => {
597 (|| -> Option<()> {
598 $body
599 Some(())
600 }())
601 };
602}