rawloader 0.37.2

A library to extract the data from camera raw formats
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

#!/usr/bin/env ruby

require 'fileutils'

BASE = File.dirname(__FILE__)
RAWBASE="https://raw.pixls.us/data/"
FILELIST = File.expand_path('filelist.sha1', File.dirname(__FILE__))
FILENUM = File.open(FILELIST).each.count

def download_file(hash, file, location)
  existhash = nil
  if File.exists?(file)
    existhash = IO.popen("sha1sum \"#{file}\"").read.split(" ")[0]
  end
  if existhash != hash
    puts "Downloading file \"#{file}\"!"
    system "curl -g -f -# \"#{RAWBASE+location.gsub(" ","%20")}\" --create-dirs -o \"#{file}\""
    newhash = IO.popen("sha1sum \"#{file}\"").read.split(" ")[0]
    if newhash != hash
      $stderr.puts "== Download checksum failed, aborting run"
      exit 2
    end
  end
end

FILES = {}
File.open(FILELIST).each_with_index do |line, i|
  lineparts = line.split("*")
  hash = lineparts[0].strip
  location = lineparts[1].strip
  filedir = File.expand_path("files", File.dirname(__FILE__))
  file = File.expand_path(hash, filedir)
  FILES[hash] = [file,location]
end

def each_sample
  FILES.each_with_index do |vals, i|
    hash, others = vals
    file, location = others
    $stderr.write "\r"
    $stderr.write "Processing #{i+1} of #{FILENUM}"
    yield hash,file,location
  end
  $stderr.write "\n"
end

$stderr.puts "== Fetching raw file samples"
each_sample do |hash, file, location|
  download_file(hash, file, location)
end

$stderr.puts "== Compiling rawloader"
system "cargo build --release --examples"
rawinfo = File.expand_path('../target/release/examples/rawinfo', BASE)

$stderr.puts "== Preparing dirs"

good = File.expand_path("good", BASE)
bad = File.expand_path("bad", BASE)
good_output = File.expand_path("good_output", BASE)
bad_output = File.expand_path("bad_output", BASE)
diff_output = File.expand_path("diff_output", BASE)
known_good_output = File.expand_path("known_good_output", BASE)
[good, bad, good_output, bad_output, diff_output].each do |dir|
  FileUtils.rm_rf dir
  FileUtils.mkdir_p dir
end

$stderr.puts "== Testing samples"
ngood = 0
nbad = 0
each_sample do |hash, file, location|
  run = IO.popen("#{rawinfo} \"#{file}\" 2>&1")
  results = run.read
  run.close
  if $? == 0
    FileUtils.ln_s file, good
    File.write(File.expand_path(hash, good_output), results)
    ngood += 1
  else
    FileUtils.ln_s file, bad
    File.write(File.expand_path(hash, bad_output), results)
    nbad += 1
  end
  $stderr.write " -- good #{ngood} bad #{nbad}"
end

$stderr.puts "== Testing known good files"
ngood = 0
nbad = 0
numfiles = Dir["#{known_good_output}/*"].count
Dir["#{known_good_output}/*"].each_with_index do |output, i|
  hash = File.basename(output)
  new_output = File.expand_path("good_output/#{hash}", BASE)
  if !File.exists? new_output
    $stderr.puts "\n!! BAD file #{hash} !!\n"
    new_output = "/dev/null"
  end
  known_output_content = File.read(output)
  new_output_content = File.read(new_output)
  if new_output_content != known_output_content
    nbad +=1
    $stderr.puts "\n!! DIFFERENT file #{hash} !!\n"
    diff_file = File.expand_path(hash, diff_output)
    system "diff -u \"#{output}\" \"#{new_output}\" > \"#{diff_file}\""
  end
  $stderr.write "\r"
  $stderr.write "Processed file #{i+1} of #{numfiles} -- good #{ngood} bad #{nbad}"
end
$stderr.write "\n"