From 0f98866eeff40d8d76c871c1adc7ce140b7b43b7 Mon Sep 17 00:00:00 2001 From: madsmpedersen <m@madsp.dk> Date: Mon, 9 Nov 2015 09:50:09 +0100 Subject: [PATCH] compare hawc2 test cases + tests --- wetb/hawc2/cmp_test_cases.py | 192 ++++++++++++++++++ wetb/hawc2/tests/test_compare_test_cases.py | 46 +++++ .../test_files/cmp_test_cases/ref/test1.dat | 4 + .../test_files/cmp_test_cases/ref/test1.sel | 16 ++ .../test_files/cmp_test_cases/ref/test2.sel | 16 ++ .../test_files/cmp_test_cases/ref/test3.dat | 4 + .../test_files/cmp_test_cases/ref/test3.sel | 16 ++ .../test_files/cmp_test_cases/ref/test4.sel | 16 ++ .../test_files/cmp_test_cases/test1/test1.dat | 4 + .../test_files/cmp_test_cases/test1/test1.sel | 16 ++ .../test_files/cmp_test_cases/test1/test2.sel | 15 ++ .../test_files/cmp_test_cases/test1/test3.dat | 4 + .../test_files/cmp_test_cases/test1/test3.sel | 16 ++ .../test_files/cmp_test_cases/test1/test4.sel | 16 ++ 14 files changed, 381 insertions(+) create mode 100644 wetb/hawc2/cmp_test_cases.py create mode 100644 wetb/hawc2/tests/test_compare_test_cases.py create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.dat create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test2.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.dat create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/ref/test4.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.dat create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test2.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.dat create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.sel create mode 100644 wetb/hawc2/tests/test_files/cmp_test_cases/test1/test4.sel diff --git a/wetb/hawc2/cmp_test_cases.py b/wetb/hawc2/cmp_test_cases.py new file mode 100644 index 0000000..9acd535 --- /dev/null +++ b/wetb/hawc2/cmp_test_cases.py @@ -0,0 +1,192 @@ +import string +import shutil +import numpy as np +import os +import filecmp + +import matplotlib.pyplot as plt + + + + + + +import unittest +import numpy as np +import re +import sys +from wetb.hawc2.Hawc2io import ReadHawc2 + + +class CompareTestCases(unittest.TestCase): + + + def compare_lines(self, ref_file, test_file, skip_first=0): + with open(ref_file) as ref: + ref_lines = ref.readlines() + with open(test_file) as test: + test_lines = test.readlines() + self.assertEqual(len(ref_lines), len(test_lines), "\nNumber of lines differs in: '%s' and '%s'" % (ref_file, test_file)) + for i, (ref_l, test_l) in enumerate(zip(ref_lines[skip_first:], test_lines[skip_first:])): + if ref_l.lower() != test_l.lower(): + diff = "".join([[" ", "^"][a != b] for a, b in zip(ref_l, test_l)]) + err_str = "%s%s%s\n\n" % (ref_l, test_l, diff) + raise AssertionError("Difference in line %d of %s\n%s" % (i, ref_file, err_str)) + + def compare_sel(self, ref_file, test_file): + self.compare_lines(ref_file, test_file, 8) + + + def compare_dat_contents(self, ref_file, test_file): + if filecmp.cmp(ref_file, test_file, shallow=False) is False: + self.compare_lines(ref_file, test_file) + + def min_tol(self, ref_data, test_data): + def error(x, a, b): + atol, rtol = x + if rtol > 0 and atol > 0 and np.allclose(b, a, rtol, atol): + return rtol + atol + else: + return 10 ** 99 + from scipy.optimize import fmin + atol, rtol = fmin(error, (1, 1), (ref_data, test_data), disp=False) + return atol, rtol + + + + + def compare_dat_plot(self, ref_file, test_file, show_plot=False, rtol=1.e-5, atol=1.e-8): + + ref = ReadHawc2(os.path.splitext(ref_file)[0]) + test = ReadHawc2(os.path.splitext(test_file)[0]) + ref_data = ref() + test_data = test() + if not np.allclose(ref_data, test_data, rtol=rtol, atol=atol): + different_sensors = [] + for i in range(ref.NrCh): + if not np.allclose(ref_data[:, i], test_data[:, i], rtol=rtol, atol=atol): + different_sensors.append(i) + path = os.path.join(os.path.dirname(test_file), "Compare", os.path.splitext(os.path.basename(ref_file))[0]) + shutil.rmtree(path, ignore_errors=True) + try: + os.mkdir(path) + except: + try: + os.mkdir(os.path.join(os.path.dirname(test_file), "Compare")) + os.mkdir(path) + except: + pass + valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) + + sys.stderr.write("%d sensors are different in datafiles\n" % len(different_sensors)) + sys.stderr.write("All close with\nAbsolute tolerance: %.6f\nRelative tolerance: %.6f\n\n" % self.min_tol(ref_data, test_data)) + + sys.stderr.write(" ".join("%-20s" % s for s in ["Mean abs error", "Mean rel error (%)", "Max abs error", "Max rel error(%)", "Sensor"]) + "\n") + abs_err = np.abs(ref_data - test_data) + mask = (np.abs(ref_data) != 0) + rel_err = abs_err[mask] / np.abs(ref_data[mask]) * 100 + err_str = ["%.6f" % e for e in [np.mean(abs_err), np.mean(rel_err), np.max(abs_err), np.max(rel_err)]] + err_str = " ".join(["%-20s" % e for e in err_str]) + sys.stderr.write("%s All data values\n" % (err_str)) + + for i in different_sensors: + abs_err = np.abs(ref_data[:, i] - test_data[:, i]) + mask = (np.abs(ref_data[:, i]) != 0) + rel_err = abs_err[mask] / np.abs(ref_data[mask, i]) * 100 + err_str = ["%.6f" % e for e in [np.mean(abs_err), np.mean(rel_err), np.max(abs_err), np.max(rel_err)]] + err_str = " ".join(["%-20s" % e for e in err_str]) + sys.stderr.write("%s %d %s [%s] %s\n" % (err_str, (i + 1), ref.ChInfo[0][i], ref.ChInfo[1][i], ref.ChInfo[2][i])) + sys.stderr.flush() + plt.cla() + plt.plot(ref_data[:, i], 'g', lw=3, label="Ref: %s [%s] %s" % (ref.ChInfo[0][i], ref.ChInfo[1][i], ref.ChInfo[2][i])) + plt.plot(test_data[:, i], 'r', lw=1, label="test: %s [%s] %s" % (test.ChInfo[0][i], test.ChInfo[1][i], test.ChInfo[2][i])) + from matplotlib.font_manager import FontProperties + fontP = FontProperties() + fontP.set_size('small') + plt.legend(loc='best', prop=fontP) + plt.axes().set_title(os.path.basename(ref_file)) + if show_plot: + plt.show() + else: + + plot_file = os.path.join(path, ("%03d_" % (i + 1)) + "".join([c for c in ref.ChInfo[0][i] if c in valid_chars]) + ".png") + plt.savefig(plot_file) + #raise AssertionError("Difference in the the values of:\n%s" % "\n".join(["%d %s" % (i + 1, ref.ChInfo[0][i]) for i in different_sensors])) + + + + def version_tag(self, filename): + re_version = re.compile(r".*_(\d*\.\d*)\.sel") + match = re.match(re_version, filename) + if match and len(match.groups()) == 1: + return match.group(1) + return "" + + def common_path(self, path1, path2): + cp = [] + for f1, f2 in zip(os.path.realpath(path1).split(os.path.sep), os.path.realpath(path2).split(os.path.sep)): + if f1 == f2: + cp.append(f1) + else: + break + return os.path.sep.join(cp) + + + def compare_file(self, ref_file, test_file, show_plot=False, rtol=1.e-5, atol=1.e-8): + try: + assert os.path.isfile(test_file), "File '%s' not found" % test_file + try: + self.compare_sel(ref_file, test_file) + self.compare_dat_plot(ref_file.replace(".sel", ".dat"), test_file.replace(".sel", ".dat"), show_plot=show_plot, rtol=rtol, atol=atol) + print ("ok\n\n\n") + except AssertionError as e: + sys.stderr.write(str(e) + "\n") + self.compare_dat_plot(ref_file.replace(".sel", ".dat"), test_file.replace(".sel", ".dat"), show_plot=show_plot, rtol=rtol, atol=atol) + print ("Data file ok\n\n\n") + + except AssertionError as e: + sys.stderr.write (str(e) + "\n\n") + + + def compare_folder(self, ref_res_path, test_res_path, ref_version_tag, test_version_tag, show_plot=False, rtol=1.e-5, atol=1.e-8): + + files = [f for f in os.listdir(ref_res_path) if f.endswith(".sel")] + common_path = self.common_path(ref_res_path, test_res_path) + for filename in files: + + self.version_tag(filename) + ref_file = os.path.join(ref_res_path, filename) + ref_version_tag = self.version_tag(filename) + test_version_tag = self.version_tag([f for f in os.listdir(test_res_path) if f.endswith(".sel")][0]) + + + + + print ("-"*50) + + try: + prefix = filename[:filename.index(ref_version_tag)] + postfix = filename[filename.index(ref_version_tag) + len(ref_version_tag):] + test_filename = [f for f in os.listdir(test_res_path) if f.startswith(prefix) and f.endswith(postfix)][0] + test_file = os.path.join(test_res_path, test_filename) + print ("Comparing %s and %s\n" % tuple(f.replace(common_path, "") for f in (ref_file, test_file))) + except IndexError: + sys.stdout.flush() + sys.stderr.write ("\nNo matching test file found for %s\n\n" % ref_file.replace(common_path, "")) + sys.stderr.flush() + continue + + self.compare_file(ref_file, test_file, show_plot, rtol, atol) + + def runTest(self): + pass + + +if __name__ == "__main__": + + ref_path = r'S:\AED\HAWC2\HAWC2_release_test_cases\version_11.4\output\res/' + test_path = r'S:\AED\HAWC2\HAWC2_release_test_cases\version_11.8w\output\res/' + # rtol: relative tolerance + # atol: absolute tolerance + # absolute(`a` - `b`) <= (`atol` + `rtol` * absolute(`b`)) + CompareTestCases().compare_folder(ref_path, test_path, "11.4", "11.8w", show_plot=False, rtol=1.e-5, atol=1.e-5) diff --git a/wetb/hawc2/tests/test_compare_test_cases.py b/wetb/hawc2/tests/test_compare_test_cases.py new file mode 100644 index 0000000..1e6005a --- /dev/null +++ b/wetb/hawc2/tests/test_compare_test_cases.py @@ -0,0 +1,46 @@ +''' +Created on 17/07/2014 + +@author: MMPE +''' +import unittest +from wetb.hawc2.cmp_test_cases import CompareTestCases +import numpy as np + +class Test(CompareTestCases): + + def setUp(self): + unittest.TestCase.setUp(self) + self.ref_path = r'test_files/cmp_test_cases/ref/' + self.test_path = r'test_files/cmp_test_cases/test1/' + + def test_compare_sel(self): + self.compare_sel(self.ref_path + 'test1.sel', self.test_path + 'test1.sel') + + def test_compare_sel_different_number_of_lines(self): + self.assertRaisesRegex(AssertionError, "16 != 15 : \nNumber of lines differs in", self.compare_sel, self.ref_path + 'test2.sel', self.test_path + 'test2.sel') + + def test_compare_sel_different_header(self): + self.compare_sel(self.ref_path + 'test3.sel', self.test_path + 'test3.sel') + + def test_compare_sel_difference(self): + self.assertRaisesRegex(AssertionError, " 2 bea1 angle deg shaft_rot angle", + self.compare_sel, self.ref_path + 'test4.sel', self.test_path + 'test4.sel') + + def test_compare_contents_dat(self): + self.compare_dat_contents(self.ref_path + 'test1.dat', self.test_path + 'test1.dat') + + def test_compare_dat_contents_difference(self): + self.assertRaisesRegex(AssertionError, " 2.00000E-02", self.compare_dat_contents, self.ref_path + 'test3.dat', self.test_path + 'test3.dat') + + #def test_compare_plot_difference(self): + # self.assertRaisesRegex(AssertionError, "Difference in the the values of:\n1 Time", self.compare_dat_plot, self.ref_path + 'test3', self.test_path + 'test3') + +# def test_compare_folder(self): +# self.compare_folder(r'test_files/ref/', r'test_files/test/', 'ref', 'test1') + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() + diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.dat b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.dat new file mode 100644 index 0000000..f45065e --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.dat @@ -0,0 +1,4 @@ + 2.00000E-02 1.14590E+00 9.54919E+00 0.00000E+00 + 4.00000E-02 2.29178E+00 9.54894E+00 0.00000E+00 + 6.00000E-02 3.43760E+00 9.54852E+00 0.00000E+00 + 8.00000E-02 4.58325E+00 9.54713E+00 0.00000E+00 diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.sel new file mode 100644 index 0000000..0662c8f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test2.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test2.sel new file mode 100644 index 0000000..0662c8f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test2.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.dat b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.dat new file mode 100644 index 0000000..f45065e --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.dat @@ -0,0 +1,4 @@ + 2.00000E-02 1.14590E+00 9.54919E+00 0.00000E+00 + 4.00000E-02 2.29178E+00 9.54894E+00 0.00000E+00 + 6.00000E-02 3.43760E+00 9.54852E+00 0.00000E+00 + 8.00000E-02 4.58325E+00 9.54713E+00 0.00000E+00 diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.sel new file mode 100644 index 0000000..0662c8f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test4.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test4.sel new file mode 100644 index 0000000..0662c8f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/ref/test4.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.dat b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.dat new file mode 100644 index 0000000..f45065e --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.dat @@ -0,0 +1,4 @@ + 2.00000E-02 1.14590E+00 9.54919E+00 0.00000E+00 + 4.00000E-02 2.29178E+00 9.54894E+00 0.00000E+00 + 6.00000E-02 3.43760E+00 9.54852E+00 0.00000E+00 + 8.00000E-02 4.58325E+00 9.54713E+00 0.00000E+00 diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.sel new file mode 100644 index 0000000..0662c8f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test2.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test2.sel new file mode 100644 index 0000000..ad2f13f --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test2.sel @@ -0,0 +1,15 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 3 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.dat b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.dat new file mode 100644 index 0000000..798d377 --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.dat @@ -0,0 +1,4 @@ + 3.00000E-02 1.14590E+00 9.54919E+00 0.00000E+00 + 4.00000E-02 2.29178E+00 9.54894E+00 0.00000E+00 + 6.00000E-02 3.43760E+00 7.54852E+00 0.00000E+00 + 8.00000E-02 4.58325E+00 9.54713E+00 0.00000E+00 diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.sel new file mode 100644 index 0000000..0183559 --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.x + Time : 14:53:52 + Date : 10:01.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.x.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle deg shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file diff --git a/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test4.sel b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test4.sel new file mode 100644 index 0000000..5512f36 --- /dev/null +++ b/wetb/hawc2/tests/test_files/cmp_test_cases/test1/test4.sel @@ -0,0 +1,16 @@ +________________________________________________________________________________________________________________________ + Version ID : HAWC2MB 11.5 + Time : 13:53:52 + Date : 10:06.2014 +________________________________________________________________________________________________________________________ + Result file : ./res/tlc_2_1_aerodyn_11.5.dat +________________________________________________________________________________________________________________________ + Scans Channels Time [sec] Format + 4 4 100.000 ASCII + + Channel Variable Description + + 1 Time s Time + 2 bea1 angle xxx shaft_rot angle + 3 bea1 angle_speed rpm shaft_rot angle speed + 4 bea1 angle deg pitch1 angle \ No newline at end of file -- GitLab