forked from lijiext/lammps
first drafts of
benchmark,py - numerical comparison of log files regression.sh - script for cron git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@10640 f3b2605a-c512-4ea7-a41b-209d697bcdaa
This commit is contained in:
rjones
parent
b22a8a7b61
commit
1144c1b3f9
|
|
@ -0,0 +1,235 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
"""
|
||||||
|
function: numerical comparisions of logs and corresponding benchmarks
|
||||||
|
usage: benchmark.py <nprocs> <njobs> <dirs>
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
from operator import itemgetter
|
||||||
|
from glob import glob
|
||||||
|
import time
|
||||||
|
import multiprocessing as mp
|
||||||
|
try:
|
||||||
|
import Queue as queue # 2.6
|
||||||
|
except ImportError:
|
||||||
|
import queue # 3.0
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### constants
|
||||||
|
#====================================================
|
||||||
|
thermo_pattern = re.compile("^Step "); # fragile
|
||||||
|
data_pattern = re.compile("\s*\d"); # fragile
|
||||||
|
fail_pattern = re.compile("FAIL");
|
||||||
|
tol = 1.e-6 # 1.e-10
|
||||||
|
arch = "openmpi"
|
||||||
|
src_path = "../src/" #relative to home
|
||||||
|
exe_path = "../"+src_path
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### date
|
||||||
|
#====================================================
|
||||||
|
def date():
|
||||||
|
return time.asctime()
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### timer
|
||||||
|
#====================================================
|
||||||
|
## NOTE these don't seem to work how I expect them to
|
||||||
|
def start():
|
||||||
|
global dt
|
||||||
|
dt = -(time.clock())
|
||||||
|
def stop():
|
||||||
|
global dt
|
||||||
|
dt += (time.clock())
|
||||||
|
return dt
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### run a benchmark
|
||||||
|
#====================================================
|
||||||
|
def run_test(test):
|
||||||
|
input = "in."+test;
|
||||||
|
log = "log."+test
|
||||||
|
stdout = "stdout."+test
|
||||||
|
ref = (glob(log+"*."+str(np)))[0];
|
||||||
|
msg = "==== comparing "+log+" with "+ref+" ====\n"
|
||||||
|
if (os.path.isfile(log)): os.remove(log)
|
||||||
|
if (os.path.isfile(stdout)): os.remove(stdout)
|
||||||
|
os.system(lmps+input+" >& "+stdout);
|
||||||
|
if (not os.path.isfile(log)) :
|
||||||
|
msg += "!!! no "+log+"\n";
|
||||||
|
msg += "!!! test "+test+" FAILED\n"
|
||||||
|
return msg
|
||||||
|
[cdict,cdata] = extract_data(log);
|
||||||
|
[bdict,bdata] = extract_data(ref);
|
||||||
|
cols = range(len(bdict))
|
||||||
|
if (len(cdata) != len(bdata)):
|
||||||
|
msg += "!!! data size "+str(len(cdata))+" does not match data "+str(len(bdata))+" in "+ref+"\n";
|
||||||
|
msg += "!!! test "+test+" FAILED\n"
|
||||||
|
return msg
|
||||||
|
fail = False
|
||||||
|
i = 0
|
||||||
|
for name in bdict:
|
||||||
|
[passing,cmsg] = compare(name,cdata[cols[i]],bdata[cols[i]]);
|
||||||
|
i += 1
|
||||||
|
msg += cmsg
|
||||||
|
if (not passing) : fail = True
|
||||||
|
if (fail) :
|
||||||
|
msg += "!!! test "+test+" FAILED\n"
|
||||||
|
else :
|
||||||
|
msg += "*** test "+test+" passed\n"
|
||||||
|
return msg
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### extract data from log file
|
||||||
|
#====================================================
|
||||||
|
def extract_data(file):
|
||||||
|
dictionary = [];
|
||||||
|
data = []
|
||||||
|
read = False
|
||||||
|
for line in open(file):
|
||||||
|
if (read and data_pattern.match(line)) :
|
||||||
|
cols = line.split();
|
||||||
|
data.append(cols)
|
||||||
|
else :
|
||||||
|
read = False
|
||||||
|
if (thermo_pattern.match(line)):
|
||||||
|
dictionary = line.split();
|
||||||
|
read = True
|
||||||
|
return [dictionary,data]
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### compare columns of current and benchmark
|
||||||
|
#====================================================
|
||||||
|
def compare(name,col1,col2):
|
||||||
|
err = 0.
|
||||||
|
norm1 = 0.
|
||||||
|
norm2 = 0.
|
||||||
|
n = len(col2)
|
||||||
|
for i in range(n):
|
||||||
|
v1 = float(col1[i])
|
||||||
|
v2 = float(col2[i])
|
||||||
|
norm1 += v1*v1
|
||||||
|
norm2 += v2*v2
|
||||||
|
dv = v1-v2
|
||||||
|
err += dv*dv
|
||||||
|
norm1 /= n
|
||||||
|
norm2 /= n
|
||||||
|
err /= n
|
||||||
|
if (norm2 > tol) :
|
||||||
|
msg = "{0:7s} relative error {1:4} wrt norm {2:7}\n".format(name,err,norm2)
|
||||||
|
else :
|
||||||
|
msg = "{0:7s} error {1:4}\n" .format(name,err)
|
||||||
|
return [(err < tol),msg];
|
||||||
|
|
||||||
|
#################################################################
|
||||||
|
class Worker(mp.Process):
|
||||||
|
def __init__(self, work_queue, result_queue):
|
||||||
|
mp.Process.__init__(self)
|
||||||
|
self.work_queue = work_queue
|
||||||
|
self.result_queue = result_queue
|
||||||
|
def run(self):
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
job = self.work_queue.get_nowait()
|
||||||
|
except queue.Empty:
|
||||||
|
break
|
||||||
|
#print(">>> starting " + str(job[1]) + " ...")
|
||||||
|
os.chdir(job[0])
|
||||||
|
start()
|
||||||
|
msg = run_test(job[1])
|
||||||
|
elapsed_time = stop()
|
||||||
|
msg += "elapsed time "+str(elapsed_time)+"\n"
|
||||||
|
os.chdir(home)
|
||||||
|
self.result_queue.put([job[1],msg])
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### parse
|
||||||
|
#====================================================
|
||||||
|
def init() :
|
||||||
|
global np, njobs, ntests, lmps, arch, home
|
||||||
|
home = os.getcwd()
|
||||||
|
if (len(sys.argv) < 4) :
|
||||||
|
print "usage: benchmark.py <nprocs> <njobs> <test_dirs>"
|
||||||
|
sys.exit(1)
|
||||||
|
np = int(sys.argv[1])
|
||||||
|
njobs = int(sys.argv[2])
|
||||||
|
lmps = "../"+src_path+"lmp_"+arch+" -in "
|
||||||
|
if (np > 1):
|
||||||
|
lmps = "mpirun -np "+str(np)+" "+lmps
|
||||||
|
else:
|
||||||
|
arch = "serial"
|
||||||
|
lmps = exe_path+"lmp_"+arch+" -in "
|
||||||
|
pool = mp.Pool(njobs)
|
||||||
|
dirs = sys.argv[3:]
|
||||||
|
tests = []
|
||||||
|
for dir in dirs:
|
||||||
|
os.chdir(dir);
|
||||||
|
for path in glob("./in.*"):
|
||||||
|
test = path[5:];
|
||||||
|
tests.append([dir,test])
|
||||||
|
os.chdir(home)
|
||||||
|
ntests = len(tests)
|
||||||
|
print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
|
||||||
|
print "start: ",date()
|
||||||
|
print "arch:",arch,
|
||||||
|
print "nprocs:",np
|
||||||
|
print "ntests:",ntests,
|
||||||
|
print "njobs:",njobs
|
||||||
|
print "relative tolerance:",tol
|
||||||
|
print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
|
||||||
|
print
|
||||||
|
return tests
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### build executable
|
||||||
|
#====================================================
|
||||||
|
def build(arch):
|
||||||
|
os.system("cd ..; svn update >& svn_update.log")
|
||||||
|
os.system("cd ../src; make no-atc >& /dev/null")
|
||||||
|
os.system("cd ../src; make clean-all >& /dev/null")
|
||||||
|
#os.system("cd ../src; make yes-all >& /dev/null")
|
||||||
|
os.system("cd ../src; make yes-dipole >& /dev/null")
|
||||||
|
sys.stdout.flush()
|
||||||
|
print "** building ",arch,"...",
|
||||||
|
os.system("cd "+src_path+"; make -j "+str(np)+" "+arch+" >& build_"+arch+".log")
|
||||||
|
if (not os.path.isfile(src_path+"lmp_"+arch)) :
|
||||||
|
print "!!! build ",arch," FAILED"
|
||||||
|
sys.exit()
|
||||||
|
else:
|
||||||
|
print "done"
|
||||||
|
print
|
||||||
|
|
||||||
|
#====================================================
|
||||||
|
### main
|
||||||
|
#====================================================
|
||||||
|
if __name__ == '__main__':
|
||||||
|
tests = init()
|
||||||
|
build(arch)
|
||||||
|
work_queue = mp.Queue()
|
||||||
|
for test in tests:
|
||||||
|
work_queue.put(test)
|
||||||
|
result_queue = mp.Queue()
|
||||||
|
nfails = 0
|
||||||
|
fail_list = []
|
||||||
|
for i in range(njobs):
|
||||||
|
w = Worker(work_queue, result_queue)
|
||||||
|
w.start()
|
||||||
|
for i in range(ntests):
|
||||||
|
[test,msg] = result_queue.get()
|
||||||
|
if (fail_pattern.search(msg)) :
|
||||||
|
nfails += 1
|
||||||
|
fail_list.append(test)
|
||||||
|
#print msg # can print only if failed
|
||||||
|
print msg # can print only if failed
|
||||||
|
#print test, " passed"
|
||||||
|
print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
|
||||||
|
print "end:",date()
|
||||||
|
if (nfails == 0):
|
||||||
|
print "*** no failures ***"
|
||||||
|
else :
|
||||||
|
print "!!!",nfails,"of",ntests,"tests failed"
|
||||||
|
for test in fail_list:
|
||||||
|
print test
|
||||||
|
print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
cd /code/lammps-atc/regression
|
||||||
|
./benchmark.py 4 12 min dipole >& latest
|
||||||
|
fail=`grep -c FAIL latest`
|
||||||
|
addrs="rjones@sandia.gov jatempl@sandia.gov jzimmer@sandia.gov sjplimp@sandia.gov akohlmey@gmail.com"
|
||||||
|
if [ $fail == 0 ] ; then
|
||||||
|
mhmail $addrs -subject "LAMMPS regression passes" < latest
|
||||||
|
else
|
||||||
|
mhmail $addrs -subject "LAMMPS regression $fail tests failed" < latest
|
||||||
|
fi
|
||||||
|
|
||||||
Loading…
Reference in New Issue