compare hawc2 test cases + tests

wetb/hawc2/cmp_test_cases.py

+import string
+import shutil
+import numpy as np
+import os
+import filecmp
+
+import matplotlib.pyplot as plt
+
+
+
+
+
+
+import unittest
+import numpy as np
+import re
+import sys
+from wetb.hawc2.Hawc2io import ReadHawc2
+
+
+class CompareTestCases(unittest.TestCase):
+
+
+    def compare_lines(self, ref_file, test_file, skip_first=0):
+        with open(ref_file) as ref:
+            ref_lines = ref.readlines()
+        with open(test_file) as test:
+            test_lines = test.readlines()
+        self.assertEqual(len(ref_lines), len(test_lines), "\nNumber of lines differs in: '%s' and '%s'" % (ref_file, test_file))
+        for i, (ref_l, test_l) in enumerate(zip(ref_lines[skip_first:], test_lines[skip_first:])):
+            if ref_l.lower() != test_l.lower():
+                diff = "".join([[" ", "^"][a != b] for a, b in zip(ref_l, test_l)])
+                err_str = "%s%s%s\n\n" % (ref_l, test_l, diff)
+                raise AssertionError("Difference in line %d of %s\n%s" % (i, ref_file, err_str))
+
+    def compare_sel(self, ref_file, test_file):
+        self.compare_lines(ref_file, test_file, 8)
+
+
+    def compare_dat_contents(self, ref_file, test_file):
+        if filecmp.cmp(ref_file, test_file, shallow=False) is False:
+            self.compare_lines(ref_file, test_file)
+
+    def min_tol(self, ref_data, test_data):
+        def error(x, a, b):
+            atol, rtol = x
+            if rtol > 0 and atol > 0 and np.allclose(b, a, rtol, atol):
+                return rtol + atol
+            else:
+                return 10 ** 99
+        from scipy.optimize import fmin
+        atol, rtol = fmin(error, (1, 1), (ref_data, test_data), disp=False)
+        return atol, rtol
+
+
+
+
+    def compare_dat_plot(self, ref_file, test_file, show_plot=False, rtol=1.e-5, atol=1.e-8):
+
+        ref = ReadHawc2(os.path.splitext(ref_file)[0])
+        test = ReadHawc2(os.path.splitext(test_file)[0])
+        ref_data = ref()
+        test_data = test()
+        if not np.allclose(ref_data, test_data, rtol=rtol, atol=atol):
+            different_sensors = []
+            for i in range(ref.NrCh):
+                if not np.allclose(ref_data[:, i], test_data[:, i], rtol=rtol, atol=atol):
+                    different_sensors.append(i)
+            path = os.path.join(os.path.dirname(test_file), "Compare", os.path.splitext(os.path.basename(ref_file))[0])
+            shutil.rmtree(path, ignore_errors=True)
+            try:
+                os.mkdir(path)
+            except:
+                try:
+                    os.mkdir(os.path.join(os.path.dirname(test_file), "Compare"))
+                    os.mkdir(path)
+                except:
+                    pass
+            valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
+
+            sys.stderr.write("%d sensors are different in datafiles\n" % len(different_sensors))
+            sys.stderr.write("All close with\nAbsolute tolerance: %.6f\nRelative tolerance: %.6f\n\n" % self.min_tol(ref_data, test_data))
+
+            sys.stderr.write("  ".join("%-20s" % s for s in ["Mean abs error", "Mean rel error (%)", "Max abs error", "Max rel error(%)", "Sensor"]) + "\n")
+            abs_err = np.abs(ref_data - test_data)
+            mask = (np.abs(ref_data) != 0)
+            rel_err = abs_err[mask] / np.abs(ref_data[mask]) * 100
+            err_str = ["%.6f" % e for e in [np.mean(abs_err), np.mean(rel_err), np.max(abs_err), np.max(rel_err)]]
+            err_str = "  ".join(["%-20s" % e  for e in err_str])
+            sys.stderr.write("%s  All data values\n" % (err_str))
+
+            for i in different_sensors:
+                abs_err = np.abs(ref_data[:, i] - test_data[:, i])
+                mask = (np.abs(ref_data[:, i]) != 0)
+                rel_err = abs_err[mask] / np.abs(ref_data[mask, i]) * 100
+                err_str = ["%.6f" % e for e in [np.mean(abs_err), np.mean(rel_err), np.max(abs_err), np.max(rel_err)]]
+                err_str = "  ".join(["%-20s" % e  for e in err_str])
+                sys.stderr.write("%s  %d %s [%s] %s\n" % (err_str, (i + 1), ref.ChInfo[0][i], ref.ChInfo[1][i], ref.ChInfo[2][i]))
+                sys.stderr.flush()
+                plt.cla()
+                plt.plot(ref_data[:, i], 'g', lw=3, label="Ref: %s [%s] %s" % (ref.ChInfo[0][i], ref.ChInfo[1][i], ref.ChInfo[2][i]))
+                plt.plot(test_data[:, i], 'r', lw=1, label="test: %s [%s] %s" % (test.ChInfo[0][i], test.ChInfo[1][i], test.ChInfo[2][i]))
+                from matplotlib.font_manager import FontProperties
+                fontP = FontProperties()
+                fontP.set_size('small')
+                plt.legend(loc='best', prop=fontP)
+                plt.axes().set_title(os.path.basename(ref_file))
+                if show_plot:
+                    plt.show()
+                else:
+
+                    plot_file = os.path.join(path, ("%03d_" % (i + 1)) + "".join([c for c in  ref.ChInfo[0][i] if c in valid_chars]) + ".png")
+                    plt.savefig(plot_file)
+            #raise AssertionError("Difference in the the values of:\n%s" % "\n".join(["%d %s" % (i + 1, ref.ChInfo[0][i]) for i in different_sensors]))
+
+
+
+    def version_tag(self, filename):
+        re_version = re.compile(r".*_(\d*\.\d*)\.sel")
+        match = re.match(re_version, filename)
+        if match and len(match.groups()) == 1:
+            return match.group(1)
+        return ""
+
+    def common_path(self, path1, path2):
+        cp = []
+        for f1, f2 in zip(os.path.realpath(path1).split(os.path.sep), os.path.realpath(path2).split(os.path.sep)):
+            if f1 == f2:
+                cp.append(f1)
+            else:
+                break
+        return os.path.sep.join(cp)
+
+
+    def compare_file(self, ref_file, test_file, show_plot=False, rtol=1.e-5, atol=1.e-8):
+        try:
+            assert os.path.isfile(test_file), "File '%s' not found" % test_file
+            try:
+                self.compare_sel(ref_file, test_file)
+                self.compare_dat_plot(ref_file.replace(".sel", ".dat"), test_file.replace(".sel", ".dat"), show_plot=show_plot, rtol=rtol, atol=atol)
+                print ("ok\n\n\n")
+            except AssertionError as e:
+                sys.stderr.write(str(e) + "\n")
+                self.compare_dat_plot(ref_file.replace(".sel", ".dat"), test_file.replace(".sel", ".dat"), show_plot=show_plot, rtol=rtol, atol=atol)
+                print ("Data file ok\n\n\n")
+
+        except AssertionError as e:
+            sys.stderr.write (str(e) + "\n\n")
+
+
+    def compare_folder(self, ref_res_path, test_res_path, ref_version_tag, test_version_tag, show_plot=False, rtol=1.e-5, atol=1.e-8):
+
+        files = [f for f in os.listdir(ref_res_path) if f.endswith(".sel")]
+        common_path = self.common_path(ref_res_path, test_res_path)
+        for filename in files:
+
+            self.version_tag(filename)
+            ref_file = os.path.join(ref_res_path, filename)
+            ref_version_tag = self.version_tag(filename)
+            test_version_tag = self.version_tag([f for f in os.listdir(test_res_path) if f.endswith(".sel")][0])
+
+
+
+
+            print ("-"*50)
+
+            try:
+                prefix = filename[:filename.index(ref_version_tag)]
+                postfix = filename[filename.index(ref_version_tag) + len(ref_version_tag):]
+                test_filename = [f for f in os.listdir(test_res_path) if f.startswith(prefix) and f.endswith(postfix)][0]
+                test_file = os.path.join(test_res_path, test_filename)
+                print ("Comparing %s and %s\n" % tuple(f.replace(common_path, "") for f in (ref_file, test_file)))
+            except IndexError:
+                sys.stdout.flush()
+                sys.stderr.write ("\nNo matching test file found for %s\n\n" % ref_file.replace(common_path, ""))
+                sys.stderr.flush()
+                continue
+
+            self.compare_file(ref_file, test_file, show_plot, rtol, atol)
+
+    def runTest(self):
+        pass
+
+
+if __name__ == "__main__":
+
+    ref_path = r'S:\AED\HAWC2\HAWC2_release_test_cases\version_11.4\output\res/'
+    test_path = r'S:\AED\HAWC2\HAWC2_release_test_cases\version_11.8w\output\res/'
+    # rtol: relative tolerance
+    # atol: absolute tolerance
+    # absolute(`a` - `b`) <= (`atol` + `rtol` * absolute(`b`))
+    CompareTestCases().compare_folder(ref_path, test_path, "11.4", "11.8w", show_plot=False, rtol=1.e-5, atol=1.e-5)

wetb/hawc2/tests/test_compare_test_cases.py

+'''
+Created on 17/07/2014
+
+@author: MMPE
+'''
+import unittest
+from wetb.hawc2.cmp_test_cases import CompareTestCases
+import numpy as np
+
+class Test(CompareTestCases):
+
+    def setUp(self):
+        unittest.TestCase.setUp(self)
+        self.ref_path = r'test_files/cmp_test_cases/ref/'
+        self.test_path = r'test_files/cmp_test_cases/test1/'
+
+    def test_compare_sel(self):
+        self.compare_sel(self.ref_path + 'test1.sel', self.test_path + 'test1.sel')
+
+    def test_compare_sel_different_number_of_lines(self):
+        self.assertRaisesRegex(AssertionError, "16 != 15 : \nNumber of lines differs in", self.compare_sel, self.ref_path + 'test2.sel', self.test_path + 'test2.sel')
+
+    def test_compare_sel_different_header(self):
+        self.compare_sel(self.ref_path + 'test3.sel', self.test_path + 'test3.sel')
+
+    def test_compare_sel_difference(self):
+        self.assertRaisesRegex(AssertionError, "     2      bea1 angle                     deg        shaft_rot angle",
+                               self.compare_sel, self.ref_path + 'test4.sel', self.test_path + 'test4.sel')
+
+    def test_compare_contents_dat(self):
+        self.compare_dat_contents(self.ref_path + 'test1.dat', self.test_path + 'test1.dat')
+
+    def test_compare_dat_contents_difference(self):
+        self.assertRaisesRegex(AssertionError, "  2.00000E-02", self.compare_dat_contents, self.ref_path + 'test3.dat', self.test_path + 'test3.dat')
+
+    #def test_compare_plot_difference(self):
+    #    self.assertRaisesRegex(AssertionError, "Difference in the the values of:\n1 Time", self.compare_dat_plot, self.ref_path + 'test3', self.test_path + 'test3')
+
+#    def test_compare_folder(self):
+#        self.compare_folder(r'test_files/ref/', r'test_files/test/', 'ref', 'test1')
+
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.testName']
+    unittest.main()
+

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.dat

+  2.00000E-02  1.14590E+00  9.54919E+00  0.00000E+00 
+  4.00000E-02  2.29178E+00  9.54894E+00  0.00000E+00 
+  6.00000E-02  3.43760E+00  9.54852E+00  0.00000E+00 
+  8.00000E-02  4.58325E+00  9.54713E+00  0.00000E+00 

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test1.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test2.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.dat

+  2.00000E-02  1.14590E+00  9.54919E+00  0.00000E+00 
+  4.00000E-02  2.29178E+00  9.54894E+00  0.00000E+00 
+  6.00000E-02  3.43760E+00  9.54852E+00  0.00000E+00 
+  8.00000E-02  4.58325E+00  9.54713E+00  0.00000E+00 

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test3.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/ref/test4.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.dat

+  2.00000E-02  1.14590E+00  9.54919E+00  0.00000E+00 
+  4.00000E-02  2.29178E+00  9.54894E+00  0.00000E+00 
+  6.00000E-02  3.43760E+00  9.54852E+00  0.00000E+00 
+  8.00000E-02  4.58325E+00  9.54713E+00  0.00000E+00 

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test1.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test2.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     3        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.dat

+  3.00000E-02  1.14590E+00  9.54919E+00  0.00000E+00 
+  4.00000E-02  2.29178E+00  9.54894E+00  0.00000E+00 
+  6.00000E-02  3.43760E+00  7.54852E+00  0.00000E+00 
+  8.00000E-02  4.58325E+00  9.54713E+00  0.00000E+00 

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test3.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.x
+                                                            Time : 14:53:52
+                                                            Date : 10:01.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.x.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     deg        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file

wetb/hawc2/tests/test_files/cmp_test_cases/test1/test4.sel

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 11.5
+                                                            Time : 13:53:52
+                                                            Date : 10:06.2014
+________________________________________________________________________________________________________________________
+  Result file : ./res/tlc_2_1_aerodyn_11.5.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+       4     4        100.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      bea1 angle                     xxx        shaft_rot angle
+     3      bea1 angle_speed               rpm        shaft_rot angle speed
+     4      bea1 angle                     deg        pitch1 angle
\ No newline at end of file