import argparse
import bench_util
import json
import os
import re
import sys
import urllib2
RANGE_RATIO_UPPER = 1.5 RANGE_RATIO_LOWER = 2.0 ERR_RATIO = 0.08 ERR_UB = 1.0 ERR_LB = 1.5
CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000',
'simple_viewport_1000x1000_angle',
'simple_viewport_1000x1000_gpu',
'simple_viewport_1000x1000_scalar_1.100000',
'simple_viewport_1000x1000_scalar_1.100000_gpu',
]
ENTRIES_TO_EXCLUDE = [
]
_GS_CLOUD_FORMAT = 'http://storage.googleapis.com/chromium-skia-gm/perfdata/%s/%s'
def compute_ranges(benches, more_benches=None):
avg = sum(benches)/len(benches)
minimum = min(benches)
maximum = max(benches)
diff = maximum - minimum
return [minimum - diff*RANGE_RATIO_LOWER - avg*ERR_RATIO - ERR_LB,
maximum + diff*RANGE_RATIO_UPPER + avg*ERR_RATIO + ERR_UB]
def create_expectations_dict(revision_data_points, builder, extra_data=None):
bench_dict = {}
for point in revision_data_points:
if (point.time_type or not point.config in CONFIGS_TO_INCLUDE):
continue
to_skip = False
for bench_substr, config_substr, builder_substr in ENTRIES_TO_EXCLUDE:
if (bench_substr in point.bench and config_substr in point.config and
builder_substr in builder):
to_skip = True
break
if to_skip:
continue
key = (point.config, point.bench)
extras = []
for idx, dataset in extra_data:
for data in dataset:
if (data.bench == point.bench and data.config == point.config and
data.time_type == point.time_type and data.per_iter_time):
extras.append((idx, data.per_iter_time))
if key in bench_dict:
raise Exception('Duplicate bench entry: ' + str(key))
bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time, extras)
return bench_dict
def get_parent_commits(start_hash, num_back):
list_commits = urllib2.urlopen(
'https://skia.googlesource.com/skia/+log/%s?format=json&n=%d' %
(start_hash, num_back))
trunc_list = list_commits.read()[4:]
json_data = json.loads(trunc_list)
return [revision['commit'] for revision in json_data['log']]
def get_file_suffixes(commit_hash, directory):
possible_files = os.listdir(directory)
prefix = 'bench_' + commit_hash + '_data_'
return [name[len(prefix):] for name in possible_files
if name.startswith(prefix)]
def download_bench_data(builder, commit_hash, suffixes, directory):
cur_files = os.listdir(directory)
count = 0
for suffix in suffixes:
file_name = 'bench_'+commit_hash+'_data_'+suffix
if file_name in cur_files:
continue
try:
src = urllib2.urlopen(_GS_CLOUD_FORMAT % (builder, file_name))
with open(os.path.join(directory, file_name), 'w') as dest:
dest.writelines(src)
count += 1
except urllib2.HTTPError:
pass
return count
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-a', '--representation_alg', default='25th',
help='bench representation algorithm to use, see bench_util.py.')
parser.add_argument(
'-b', '--builder', required=True,
help='name of the builder whose bench ranges we are computing.')
parser.add_argument(
'-d', '--input_dir', required=True,
help='a directory containing bench data files.')
parser.add_argument(
'-o', '--output_file', required=True,
help='file path and name for storing the output bench expectations.')
parser.add_argument(
'-r', '--git_revision', required=True,
help='the git hash to indicate the revision of input data to use.')
parser.add_argument(
'-t', '--back_track', required=False, default=10,
help='the number of commit hashes backwards to look to include' +
'in the calculations.')
parser.add_argument(
'-m', '--max_commits', required=False, default=1,
help='the number of commit hashes to include in the calculations.')
args = parser.parse_args()
builder = args.builder
data_points = bench_util.parse_skp_bench_data(
args.input_dir, args.git_revision, args.representation_alg)
parent_commits = get_parent_commits(args.git_revision, args.back_track)
print "Using commits: {}".format(parent_commits)
suffixes = get_file_suffixes(args.git_revision, args.input_dir)
print "Using suffixes: {}".format(suffixes)
downloaded_commits = []
for idx, commit in enumerate(parent_commits):
num_downloaded = download_bench_data(
builder, commit, suffixes, args.input_dir)
if num_downloaded > 0:
downloaded_commits.append((num_downloaded, idx, commit))
if len(downloaded_commits) < args.max_commits:
print ('Less than desired number of commits found. Please increase'
'--back_track in later runs')
trunc_commits = sorted(downloaded_commits, reverse=True)[:args.max_commits]
extra_data = []
for _, idx, commit in trunc_commits:
extra_data.append((idx, bench_util.parse_skp_bench_data(
args.input_dir, commit, args.representation_alg)))
expectations_dict = create_expectations_dict(data_points, builder,
extra_data)
out_lines = []
keys = expectations_dict.keys()
keys.sort()
for (config, bench) in keys:
(expected, lower_bound, upper_bound) = expectations_dict[(config, bench)]
out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,'
'%(expected)s,%(lower_bound)s,%(upper_bound)s' % {
'bench': bench,
'config': config,
'builder': builder,
'representation': args.representation_alg,
'expected': expected,
'lower_bound': lower_bound,
'upper_bound': upper_bound})
with open(args.output_file, 'w') as file_handle:
file_handle.write('\n'.join(out_lines))
if __name__ == "__main__":
main()