import sys
import argparse
import subprocess
import itertools
import reimport os
import math
from threading import Timer, Thread
import thread, time
from platform import system
import numpy as np
from datetime import datetime
import errorHandler
from fftPerformanceTesting import *
from performanceUtility import timeout, log, generate235Radices
IAM = 'FFT'
TIMOUT_VAL = 900
devicevalues = ['g', 'c']
layoutvalues = ['cp', 'ci']
placevalues = ['in', 'out']
precisionvalues = ['single', 'double']
libraryvalues = ['clFFT','cuFFT']
pow10 = '1-9,10-90:10,100-900:100,1000-9000:1000,10000-90000:10000,100000-900000:100000,1000000-9000000:1000000'
parser = argparse.ArgumentParser(description='Measure performance of the clFFT library')
parser.add_argument('--device',
dest='device', default='g',
help='device(s) to run on; may be a comma-delimited list. choices are ' + str(devicevalues) + '. (default gpu)')
parser.add_argument('-b', '--batchsize',
dest='batchSize', default='1',
help='number of FFTs to perform with one invocation of the client. the special value \'adapt\' may be used to adjust the batch size on a per-transform basis to the maximum problem size possible on the device. (default 1)'.format(pow10))
parser.add_argument('-a', '--adaptivemax',
dest='constProbSize', default='-1',
help='Max problem size that you want to maintain across the invocations of client with different lengths. This is adaptive and adjusts itself automtically.'.format(pow10))
parser.add_argument('-x', '--lengthx',
dest='lengthx', default='1',
help='length(s) of x to test; must be factors of 1, 2, 3, or 5 with clFFT; may be a range or a comma-delimited list. e.g., 16-128 or 1200 or 16,2048-32768 (default 1)')
parser.add_argument('-y', '--lengthy',
dest='lengthy', default='1',
help='length(s) of y to test; must be factors of 1, 2, 3, or 5 with clFFT; may be a range or a comma-delimited list. e.g., 16-128 or 1200 or 16,32768 (default 1)')
parser.add_argument('-z', '--lengthz',
dest='lengthz', default='1',
help='length(s) of z to test; must be factors of 1, 2, 3, or 5 with clFFT; may be a range or a comma-delimited list. e.g., 16-128 or 1200 or 16,32768 (default 1)')
parser.add_argument('-reps',
dest='reps', default='10',
help='Number of repetitions (default 10)')
parser.add_argument('-prime_factor', '--prime_factor',
dest='prime_factor', default='2',
help='only test the prime factors within the specified range of lengthx/y/z. Select from 2,3,5, and 7. Example: -prime_factor 2,3')
parser.add_argument('-test_count', '--test_count',
dest='test_count', default='100',
help='Number of tests to perform')
parser.add_argument('--problemsize',
dest='problemsize', default=None)
parser.add_argument('-i', '--inputlayout',
dest='inputlayout', default='1',
help=' 1. interleaved (default) 2. planar 3. hermitian interleaved 4. hermitian planar 5. real' )
parser.add_argument('-o', '--outputlayout',
dest='outputlayout', default='1',
help=' 1. interleaved (default) 2. planar 3. hermitian interleaved 4. hermitian planar 5. real' )
parser.add_argument('--placeness',
dest='placeness', default='in',
help='Choices are ' + str(placevalues) + '. in = in place, out = out of place (default in)')
parser.add_argument('-r', '--precision',
dest='precision', default='single',
help='Choices are ' + str(precisionvalues) + '. (default single)')
parser.add_argument('--library',
dest='library', default='clFFT', choices=libraryvalues,
help='indicates the library to use for testing on this run')
parser.add_argument('--label',
dest='label', default=None,
help='a label to be associated with all transforms performed in this run. if LABEL includes any spaces, it must be in \"double quotes\". note that the label is not saved to an .ini file. e.g., --label cayman may indicate that a test was performed on a cayman card or --label \"Windows 32\" may indicate that the test was performed on Windows 32')
parser.add_argument('--ini',
dest='iniFilename', default=None,
help='use the parameters in the named .ini file instead of the command line parameters.')
parser.add_argument('--tablefile',
dest='tableOutputFilename', default=None,
help='save the results to a plaintext table with the file name indicated. this can be used with plotPerformance.py to generate graphs of the data (default: table prints to screen)')
parser.add_argument('--prefix',
dest='prefix', default='./',
help='Path where the library client is located (default current directory)')
args = parser.parse_args()
label = str(args.label)
subprocess.call('mkdir perfLog', shell = True)
logfile = os.path.join('perfLog', (label+'-'+'fftMeasurePerfLog.txt'))
def printLog(txt):
print txt
log(logfile, txt)
printLog("=========================MEASURE PERFORMANCE START===========================")
printLog("Process id of Measure Performance:"+str(os.getpid()))
currCommandProcess = None
printLog('Executing measure performance for label: '+str(label))
@timeout(1, "fileName") def checkTimeOutPut2(args):
global currCommandProcess
currCommandProcess = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
printLog("Curr Command Process id = "+str(currCommandProcess.pid))
ret = currCommandProcess.communicate()
if(ret[0] == None or ret[0] == ''):
errCode = currCommandProcess.poll()
raise subprocess.CalledProcessError(errCode, args, output=ret[1])
return ret[0]
def checkTimeOutPut(args):
t = None
global currCommandProcess
global stde
global stdo
stde = None
stdo = None
def executeCommand():
global currCommandProcess
global stdo
global stde
try:
stdo, stde = currCommandProcess.communicate()
printLog('stdout:\n'+str(stdo))
printLog('stderr:\n'+str(stde))
except:
printLog("ERROR: UNKNOWN Exception - +checkWinTimeOutPut()::executeCommand()")
currCommandProcess = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True)
thread = Thread(target=executeCommand)
thread.start()
thread.join(TIMOUT_VAL) if thread.is_alive():
printLog('ERROR: Killing the process - terminating thread because it is taking too much of time to execute')
currCommandProcess.kill()
printLog('ERROR: Timed out exception')
raise errorHandler.ApplicationException(__file__, errorHandler.TIME_OUT)
if stdo == "" or stdo==None:
errCode = currCommandProcess.poll()
printLog('ERROR: @@@@@Raising Called processor exception')
raise subprocess.CalledProcessError(errCode, args, output=stde)
return stdo
if args.batchSize.count('pow10'):
args.batchSize = pow10
args.batchSize = args.batchSize.split(',')
args.constProbSize = int(args.constProbSize.split(',')[0])
args.device = args.device.split(',')
args.lengthx = args.lengthx.split(',')
args.lengthy = args.lengthy.split(',')
args.lengthz = args.lengthz.split(',')
args.prime_factor = args.prime_factor.split(',')
if args.problemsize:
args.problemsize = args.problemsize.split(',')
args.inputlayout = args.inputlayout.split(',')
args.outputlayout = args.outputlayout.split(',')
args.placeness = args.placeness.split(',')
args.precision = args.precision.split(',')
printLog('Executing for label: '+str(args.label))
if ( args.batchSize.count('max') or args.batchSize.count('adapt') )and len(args.batchSize) > 1:
printLog('ERROR: --batchsize max must not be in a comma delimited list')
quit()
for n in args.precision:
if n != 'single' and n != 'double':
printLog('ERROR: invalid value for precision')
quit()
def isPrime(n):
import math
n = abs(n)
i = 2
while i <= math.sqrt(n):
if n%i == 0:
return False
i += 1
return True
def findFactors(number):
iter_space = range(1, number+1)
prime_factor_list = []
for curr_iter in iter_space:
if isPrime(curr_iter) == True:
if number%curr_iter == 0:
prime_factor_list.append(curr_iter)
return prime_factor_list
def factor(num):
if num == 1:
return [1]
i = 2
limit = num**0.5
while i <= limit:
if num % i == 0:
ret = factor(num/i)
ret.append(i)
return ret
i += 1
return [num]
def validateFactors(flist):
ref_list = [1,2,3,5]
if flist==ref_list:
return True
if len(flist) > len(ref_list):
return False
for felement in flist:
if ref_list.count(felement) != 1:
return False
return True
def validate_number_for_1235(num):
if num == 0:
return True
set1235 = set([1,2,3,5])
setPrimeFactors = set(factor(num))
setPrimeFactors = setPrimeFactors | set1235 if setPrimeFactors == set1235:
return True
return False
def getValidNumbersInRange(rlist):
valid_number_list = []
for relement in rlist:
prime_factors = findFactors(relement)
if validateFactors(prime_factors) == True:
valid_number_list.append(relement)
return valid_number_list
def get_next_num_with_1235_factors(start):
start+=1
while not validateFactors(findFactors(start)):
start+=1
return start
def check_number_for_1235_factors(number):
factors = findFactors(number)
if not validateFactors(factors):
printLog("ERROR: --{0} must have only 1,2,3,5 as factors")
return False
return True
def check_for_1235_factors(values, option):
for n in values:
for m in n.replace('-',',').split(','):
if not validate_number_for_1235(int(m)):
print 'ERROR: --{0} must specify number with only 1,2,3,5 as factors'.format(option)
quit()
if args.library == 'clFFT':
check_for_1235_factors(args.lengthx, 'lengthx')
check_for_1235_factors(args.lengthy, 'lengthy')
check_for_1235_factors(args.lengthz, 'lengthz')
if not os.path.isfile(args.prefix+executable(args.library)):
printLog("ERROR: Could not find client named {0}".format(executable(args.library)))
quit()
def get235RadicesNumberInRange(minimum, maximum):
if minimum == 0 and maximum == 0:
return [0]
numbers = generate235Radices(maximum)
minIndex = numbers.index(minimum)
maxIndex = numbers.index(maximum)
return numbers[minIndex:maxIndex+1]
class Range:
def __init__(self, ranges, defaultStep='+1'):
self.expanded = []
for thisRange in ranges:
if thisRange != 'max' and thisRange != 'adapt' :
if thisRange.count(':'):
self._stepAmount = thisRange.split(':')[1]
else:
self._stepAmount = defaultStep
thisRange = thisRange.split(':')[0]
if self._stepAmount.count('x'):
self._stepper = '_mult'
self._stepAmount = self._stepAmount.lstrip('+x')
self._stepAmount = int(self._stepAmount)
elif self._stepAmount.count('l'):
self._stepper = '_next_num_with_1235_factor'
self._stepAmount = 0
else:
self._stepper = '_add'
self._stepAmount = self._stepAmount.lstrip('+x')
self._stepAmount = int(self._stepAmount)
if thisRange.count('-'):
self.begin = int(thisRange.split('-')[0])
self.end = int(thisRange.split('-')[1])
else:
self.begin = int(thisRange.split('-')[0])
self.end = int(thisRange.split('-')[0])
self.current = self.begin
if thisRange == 'max':
self.expanded = self.expanded + ['max']
elif thisRange == 'adapt':
self.expanded = self.expanded + ['adapt']
elif self.begin == 0 and self._stepper == '_mult':
self.expanded = self.expanded + [0]
else:
if self._stepper == '_next_num_with_1235_factor':
self.expanded = self.expanded + get235RadicesNumberInRange(self.current, self.end)
else:
while self.current <= self.end:
self.expanded = self.expanded + [self.current]
self._step()
self.expanded = list(set(self.expanded))
self.expanded.sort()
def _step(self):
getattr(self, self._stepper)()
def _mult(self):
self.current = self.current * self._stepAmount
def _add(self):
self.current = self.current + self._stepAmount
def _next_num_with_1235_factor(self):
self.current = get_next_num_with_1235_factors(self.current)
args.batchSize = Range(args.batchSize).expanded
args.lengthx = Range(args.lengthx, 'l').expanded
args.lengthy = Range(args.lengthy, 'l').expanded
args.lengthz = Range(args.lengthz, 'l').expanded
def create_prime_factors(args,input_list):
powers2=[1]
powers3=[1]
powers5=[1]
powers7=[1]
if '2' in args.prime_factor:
powers2+=[2**x for x in range(1,int(math.floor(math.log(max(input_list),2)+1)))]
if '3' in args.prime_factor:
powers3+=[3**x for x in range(1,int(math.floor(math.log(max(input_list),3)+1)))]
if '5' in args.prime_factor:
powers5+=[5**x for x in range(1,int(math.floor(math.log(max(input_list),5)+1)))]
if '7' in args.prime_factor:
powers7+=[7**x for x in range(1,int(math.floor(math.log(max(input_list),7)+1)))]
xlist=[]
for i in powers2:
for j in powers3:
for k in powers5:
for l in powers7:
dummy=int(i)*int(j)*int(k)*int(l)
if(dummy<=max(input_list)) and (dummy>=min(input_list)):
xlist.append(dummy)
xlist=sorted(xlist)
xlist=xlist[:int(args.test_count)] return xlist
args.lengthx=create_prime_factors(args,args.lengthx)
args.lengthy=create_prime_factors(args,args.lengthy)
args.lengthz=create_prime_factors(args,args.lengthz)
if args.problemsize and args.problemsize[0] != 'None':
i = 0
while i < len(args.problemsize):
args.problemsize[i] = args.problemsize[i].split(':')
args.problemsize[i][0] = args.problemsize[i][0].split('x')
i = i+1
if args.lengthy[0]==1:
args.lengthy=[1]*len(args.lengthx)
if args.lengthz[0]==1:
args.lengthz=[1]*len(args.lengthx)
dummy=[args.batchSize[0]]*len(args.lengthx)
problem_size_combinations=zip(args.lengthx,args.lengthy,args.lengthz,dummy)
manual_test_combinations = []
if args.problemsize and args.problemsize[0] != 'None':
for n in args.problemsize:
x = []
y = []
z = []
batch = []
x.append(int(n[0][0]))
if len(n[0]) >= 2:
y.append(int(n[0][1]))
else:
y.append(1)
if len(n[0]) >= 3:
z.append(int(n[0][2]))
else:
z.append(1)
if len(n) > 1:
batch.append(int(n[1]))
else:
batch.append(1)
combos = itertools.product(x, y, z, batch)
combos = list(itertools.islice(combos, None))
for n in combos:
manual_test_combinations.append(n)
problem_size_combinations = problem_size_combinations + manual_test_combinations
test_combinations = itertools.product(problem_size_combinations, args.device, args.inputlayout, args.outputlayout, args.placeness, args.precision)
test_combinations = list(itertools.islice(test_combinations, None))
test_combinations = [TestCombination(params[0][0], params[0][1], params[0][2], params[0][3], params[1], params[2], params[3], params[4], params[5], args.label) for params in test_combinations]
if args.iniFilename != None:
array=np.genfromtxt(args.iniFilename, names=True, delimiter=',', dtype=None)
test_combinations = [TestCombination(params[0],params[1], params[2], params[3], params[4],params[5],params[6],params[7],params[8],args.label) for params in array]
result = []
if args.tableOutputFilename == None:
if args.library == 'cuFFT':
args.tableOutputFilename = 'cuFFT_' + 'x_'+ str(args.lengthx[0]) + '_y_'+str(args.lengthy[0])+'_z_'+str(args.lengthz[0])+'_'+str(args.precision[0]) +'_'+datetime.now().isoformat().replace(':','.') + '.txt'
elif args.library=='clFFT':
args.tableOutputFilename = 'clFFT_' + 'x_'+ str(args.lengthx[0]) + '_y_'+str(args.lengthy[0])+'_z_'+str(args.lengthz[0])+'_'+str(args.precision[0])+ '_'+datetime.now().isoformat().replace(':','.') + '.txt'
else:
if os.path.isfile(args.tableOutputFilename):
oldname = args.tableOutputFilename
args.tableOutputFilename = args.tableOutputFilename + datetime.now().isoformat().replace(':','.')
message = 'A file with the name ' + oldname + ' already exists. Changing filename to ' + args.tableOutputFilename
printLog(message)
printLog('table header---->'+ str(tableHeader))
table = open(args.tableOutputFilename, 'w')
table.write(tableHeader + '\n')
table.flush()
if args.constProbSize == -1:
args.constProbSize = maxBatchSize(1, 1, 1, args.inputlayout[0], args.precision[0], executable(args.library), '-' + args.device[0])
args.constProbSize = int(args.constProbSize)
printLog('Total combinations = '+str(len(test_combinations)))
vi = 0
for params in test_combinations:
if vi>=int(args.test_count):
break
vi = vi+1
printLog("")
printLog('preparing command: '+ str(vi))
device = params.device
lengthx = str(params.x)
lengthy = str(params.y)
lengthz = str(params.z)
inlayout=str(params.inlayout)
outlayout=str(params.outlayout)
prefix=str(args.prefix)
if params.batchsize == 'max':
batchSize = maxBatchSize(lengthx, lengthy, lengthz, params.inlayout, params.precision, executable(args.library), '-' + device)
elif params.batchsize == 'adapt':
batchSize = str(args.constProbSize/(int(lengthx)*int(lengthy)*int(lengthz)))
else:
batchSize = str(params.batchsize)
if params.placeness == 'inplace' or params.placeness == 'in':
placeness = ''
elif params.placeness == 'outofplace' or params.placeness == 'out':
placeness = '--outPlace'
else:
printLog('ERROR: invalid value for placeness when assembling client command')
if params.precision == 'single':
precision = ''
elif params.precision == 'double':
precision = '--double'
else:
printLog('ERROR: invalid value for precision when assembling client command')
if args.library == 'clFFT':
arguments = [prefix+ executable(args.library),
'-' + device,
'-x', lengthx,
'-y', lengthy,
'-z', lengthz,
'--batchSize', batchSize,
'--inLayout', inlayout,
'--outLayout',outlayout,
placeness,
precision,
'-p', args.reps]
elif args.library == 'cuFFT':
if inlayout[0]=='1' and outlayout[0]=='1':
cuFFT_type='1'
elif inlayout[0]=='3' and outlayout[0]=='5':
cuFFT_type='3'
elif inlayout[0]=='5' and outlayout[0]=='3':
cuFFT_type='2'
else:
print"Wrong input/outputlayout. Only C2C/R2C/C2R are supported for Cuda"
exit()
arguments=[prefix+executable(args.library),
'-x', lengthx,
'-y', lengthy,
'-z', lengthz,
'-b', batchSize,
'-p',args.reps,
'-d',str(int(args.precision[0]=='double')),
'-type',cuFFT_type]
writeline = True
try:
arguments=' '.join(arguments)
printLog('Executing Command: '+str(arguments))
output = checkTimeOutPut(arguments)
output = output.split(os.linesep);
printLog('Execution Successfull---------------\n')
except errorHandler.ApplicationException as ae:
writeline = False
printLog('ERROR: Command is taking too much of time '+ae.message+'\n'+'Command: \n'+str(arguments))
continue
except subprocess.CalledProcessError as clientCrash:
print 'Command execution failure--->'
if clientCrash.output.count('CLFFT_INVALID_BUFFER_SIZE'):
writeline = False
printLog('Omitting line from table - problem is too large')
else:
writeline = False
printLog('ERROR: client crash. Please report the following error message (with \'CLFFT_*\' error code, if given, and the parameters used to invoke measurePerformance.py) \n'+clientCrash.output+'\n')
printLog('IN ORIGINAL WE CALL QUIT HERE - 1\n')
continue
for x in output:
if x.count('out of memory'):
writeline = False
printLog('ERROR: Omitting line from table - problem is too large')
if writeline:
try:
if args.library == 'cuFFT':
output = itertools.ifilter( lambda x: x.count('gflops'), output)
else:
output = itertools.ifilter( lambda x: x.count('gflops'), output)
output = list(itertools.islice(output, None))
thisResult = re.search('\d+\.*\d*e*-*\d*$', output[-1])
if args.library == 'cuFFT':
thisResult = re.search('[-+]?\d*\.\d+|\d+$', output[-1])
thisResult = float(thisResult.group(0))
thisResult = (params.x, params.y, params.z, batchSize, params.device, params.inlayout, params.outlayout, params.placeness, params.precision, params.label, thisResult)
outputRow = ''
for x in thisResult:
outputRow = outputRow + str(x) + ','
outputRow = outputRow.rstrip(',')
table.write(outputRow + '\n')
table.flush()
except:
printLog('ERROR: Exception occurs in GFLOP parsing')
else:
if(len(output) > 0):
if output[0].find('nan') or output[0].find('inf'):
printLog( 'WARNING: output from client was funky for this run. skipping table row')
else:
prinLog('ERROR: output from client makes no sense')
printLog(str(output[0]))
printLog('IN ORIGINAL WE CALL QUIT HERE - 2\n')
else:
prinLog('ERROR: output from client makes no sense')
printLog("=========================MEASURE PERFORMANCE ENDS===========================\n")