HAWC2 LogFile reader + tests

wetb/hawc2/log_file.py

+'''
+Created on 18/11/2015
+
+@author: MMPE
+'''
+import os
+from wetb.hawc2.htc_file import HTCFile
+from collections import OrderedDict
+MISSING = "Log file cannot be found"
+PENDING = "Simulation not started yet"
+INITIALIZATION = 'Initializing simulation'
+GENERATING_TURBULENCE = "Generating turbulence"
+SIMULATING = "Simulating"
+ABORTED = ""
+DONE = "Simulation succeded"
+INITIALIZATION_ERROR = "Initialization error"
+SIMULATION_ERROR = "Simulation error"
+ERROR = "Error"
+
+def is_file_open(filename):
+    try:
+        os.rename(filename, filename + "_")
+        os.rename(filename + "_", filename)
+        return False
+    except OSError as e:
+        if "The process cannot access the file because it is being used by another process" not in str(e):
+            raise
+
+        if os.path.isfile(filename + "_"):
+            os.remove(filename + "_")
+        return True
+
+class LogFile(object):
+    _status = (0, MISSING)
+    def __init__(self, log_filename, time_stop):
+        self.filename = log_filename
+        self.time_stop = time_stop
+        self.position = 0
+
+    def clear(self):
+        os.makedirs(os.path.dirname(self.filename), exist_ok=True)
+        with open(self.filename, 'w'):
+            pass
+        self.position = 0
+
+    def status(self):
+        if not os.path.isfile(self.filename):
+            self._status = (0, MISSING, [])
+        else:
+            if self._status[1] == MISSING:
+                self._status = (0, PENDING, [])
+            with open(self.filename) as fid:
+                fid.seek(self.position)
+                txt = fid.read()
+            self.position += len(txt)
+            if self._status[1] == PENDING and self.position > 0:
+                self._status = (0, INITIALIZATION, [])
+            error_lst = self._status[2]
+            if len(txt) > 0:
+                if self._status[1] == INITIALIZATION or self._status[1] == GENERATING_TURBULENCE:
+                    init_txt, *rest = txt.split("Starting simulation")
+                    if "*** ERROR ***" in init_txt:
+                        error_lst.extend([l for l in init_txt.strip().split("\n") if "ERROR" in l])
+                    if "Turbulence generation starts" in init_txt[init_txt.strip().rfind("\n"):]:
+                        self._status = (0, GENERATING_TURBULENCE, error_lst)
+                    if rest:
+                        txt = rest[0]
+                        self._status = (0, SIMULATING, error_lst)
+                if self._status[1] == SIMULATING:
+
+                    simulation_txt, *rest = txt.split('Elapsed time')
+                    if "*** ERROR ***" in simulation_txt:
+                        error_lst.extend([l for l in simulation_txt.strip().split("\n") if "ERROR" in l])
+                    i1 = simulation_txt.rfind("Global time")
+                    i2 = simulation_txt[:i1].rfind('Global time')
+                    time_line = simulation_txt[i1:]
+                    try:
+                        time = float(time_line[time_line.index('=') + 1:time_line.index('Iter')])
+                        self._status = (int(100 * time // self.time_stop), SIMULATING, error_lst)
+                    except:
+                        self._status = (self._status[0], SIMULATING, error_lst)
+                    if rest:
+                        self._status = (100, DONE, error_lst)
+            #return self._status
+
+        error_lst = self._status[2]
+        error_dict = OrderedDict()
+        for error in error_lst:
+            error_dict[error] = error_dict.get(error, 0) + 1
+        error_lst = [("%d x %s" % (v, k), k)[v == 1] for k, v in error_dict.items()]
+
+        return (self._status[0], self._status[1], error_lst)
+

wetb/hawc2/tests/test_files/logfiles/finish_turbulencegeneration.log

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 12.2
+  Log file output
+                                                            Time : 07:18:14
+                                                            Date : 20:11.2015
+________________________________________________________________________________________________________________________
+ Simulation commands read with succes
+ Orientation input commands read with succes
+ constraint input commands read with succes
+ Topologi commands read with succes
+ Mann commands read with succes
+ Wind commands read with succes
+ DLL commands read with succes
+ output commands read with succes
+ Output commands read
+ Initialization of structure
+ Initialization of wind
+ Initialization of Mann turbulence
+ Opening turbulence file ./turb/envisionpp2b_20150517_2310_s1004u.bin
+ Turbulence file ./turb/envisionpp2b_20150517_2310_s1004u.bin does not exist
+ Turbulence generation starts ...
+ Turbulence generation finished, output starts ...
+ Mann turb. box shorter than required - turbulence is reused after every    600.14 seconds
+ In Mann module:
+ Scale factor u=   1.00000000000000     
+ Scale factor v=   1.00000000000000     
+ Scale factor w=   1.00000000000000     
+ Starting simulation
+ Global time =   2.000000000000000E-002 Iter =            0
+ Global time =   4.000000000000000E-002 Iter =            0
+ Global time =   6.000000000000000E-002 Iter =            0
+ Global time =   8.000000000000000E-002 Iter =            0
+ Global time =   0.100000000000000      Iter =            0
+ Global time =   0.120000000000000      Iter =            0
+ Global time =   0.140000000000000      Iter =            0
+ Global time =   0.160000000000000      Iter =            0
+ Global time =   0.180000000000000      Iter =            0
+ Global time =   0.200000000000000      Iter =            0
+ Global time =   0.220000000000000      Iter =            0
+ Global time =   0.240000000000000      Iter =            0
+ Global time =   0.260000000000000      Iter =            0
+ Global time =   0.280000000000000      Iter =            0
+ Global time =   0.300000000000000      Iter =            0
+ Global time =   0.320000000000000      Iter =            0
+ Global time =   0.340000000000000      Iter =            0
+ Global time =   0.360000000000000      Iter =            0
+ Global time =   0.380000000000000      Iter =            0
+ Global time =   0.400000000000000      Iter =            0
+ Global time =   0.420000000000000      Iter =            0
+ Global time =   0.440000000000000      Iter =            0
+ Global time =   0.460000000000000      Iter =            0
+ Global time =   0.480000000000000      Iter =            0
+ Global time =   0.500000000000000      Iter =            0
+ Global time =   0.520000000000000      Iter =            0
+ Global time =   0.540000000000000      Iter =            0
+ Global time =   0.560000000000000      Iter =            0
+ Global time =   0.580000000000000      Iter =            0
+ Global time =   0.600000000000000      Iter =            0
+ Global time =   0.620000000000000      Iter =            0
+ Global time =   0.640000000000000      Iter =            0
+ Global time =   0.660000000000000      Iter =            0
+ Global time =   0.680000000000000      Iter =            0
+ Global time =   0.700000000000000      Iter =            0
+ Global time =   0.720000000000000      Iter =            0
+ Global time =   0.740000000000000      Iter =            0
+ Global time =   0.760000000000000      Iter =            0
+ Global time =   0.780000000000000      Iter =            0
+ Global time =   0.800000000000000      Iter =            0
+ Global time =   0.820000000000000      Iter =            0
+ Global time =   0.840000000000000      Iter =            0
+ Global time =   0.860000000000000      Iter =            0
+ Global time =   0.880000000000000      Iter =            0
+ Global time =   0.900000000000000      Iter =            0
+ Global time =   0.920000000000000      Iter =            0
+ Global time =   0.940000000000000      Iter =            0
+ Global time =   0.960000000000000      Iter =            0
+ Global time =   0.980000000000000      Iter =            0
+ Global time =    1.00000000000000      Iter =            0
+ Global time =    1.02000000000000      Iter =            0
+ Global time =    1.04000000000000      Iter =            0
+ Global time =    1.06000000000000      Iter =            0
+ Global time =    1.08000000000000      Iter =            0
+ Global time =    1.10000000000000      Iter =            0
+ Global time =    1.12000000000000      Iter =            0
+ Global time =    1.14000000000000      Iter =            0
+ Global time =    1.16000000000000      Iter =            0
+ Global time =    1.18000000000000      Iter =            0
+ Global time =    1.20000000000000      Iter =            0
+ Global time =    1.22000000000000      Iter =            0
+ Global time =    1.24000000000000      Iter =            0
+ Global time =    1.26000000000000      Iter =            0
+ Global time =    1.28000000000000      Iter =            0
+ Global time =    1.30000000000000      Iter =            0
+ Global time =    1.32000000000000      Iter =            0
+ Global time =    1.34000000000000      Iter =            0
+ Global time =    1.36000000000000      Iter =            0
+ Global time =    1.38000000000000      Iter =            0
+ Global time =    1.40000000000000      Iter =            0
+ Global time =    1.42000000000000      Iter =            0
+ Global time =    1.44000000000000      Iter =            0
+ Global time =    1.46000000000000      Iter =            0
+ Global time =    1.48000000000000      Iter =            0
+ Global time =    1.50000000000000      Iter =            0
+ Global time =    1.52000000000000      Iter =            0
+ Global time =    1.54000000000000      Iter =            0
+ Global time =    1.56000000000000      Iter =            0
+ Global time =    1.58000000000000      Iter =            0
+ Global time =    1.60000000000000      Iter =            0
+ Global time =    1.62000000000000      Iter =            0
+ Global time =    1.64000000000000      Iter =            0
+ Global time =    1.66000000000000      Iter =            0
+ Global time =    1.68000000000000      Iter =            0
+ Global time =    1.70000000000000      Iter =            0
+ Global time =    1.72000000000000      Iter =            0
+ Global time =    1.74000000000000      Iter =            0
+ Global time =    1.76000000000000      Iter =            0
+ Global time =    1.78000000000000      Iter =            0
+ Global time =    1.80000000000000      Iter =            0
+ Global time =    1.82000000000000      Iter =            0
+ Global time =    1.84000000000000      Iter =            0
+ Global time =    1.86000000000000      Iter =            0
+ Global time =    1.88000000000000      Iter =            0
+ Global time =    1.90000000000000      Iter =            0
+ Global time =    1.92000000000000      Iter =            0
+ Global time =    1.94000000000000      Iter =            0
+ Global time =    1.96000000000000      Iter =            0
+ Global time =    1.98000000000000      Iter =            0
+ Global time =    2.00000000000000      Iter =            0
+ Elapsed time :   0.8062344    

wetb/hawc2/tests/test_files/logfiles/test.log

+hallo er en test
+og en til
+jaja
+nejnej
+jaja
+hej
+hallohallohallohej
+hej
+hej
+hallohallohallohej
+hej
+hej
+sdf
+sdf
+sdfsdf
\ No newline at end of file

wetb/hawc2/tests/test_files/logfiles/turbulence_generation.log

+________________________________________________________________________________________________________________________
+  Version ID : HAWC2MB 12.2
+  Log file output
+                                                            Time : 07:18:14
+                                                            Date : 20:11.2015
+________________________________________________________________________________________________________________________
+ Simulation commands read with succes
+ Orientation input commands read with succes
+ constraint input commands read with succes
+ Topologi commands read with succes
+ Mann commands read with succes
+ Wind commands read with succes
+ DLL commands read with succes
+ output commands read with succes
+ Output commands read
+ Initialization of structure
+ Initialization of wind
+ Initialization of Mann turbulence
+ Opening turbulence file ./turb/envisionpp2b_20150517_2310_s1004u.bin
+ Turbulence file ./turb/envisionpp2b_20150517_2310_s1004u.bin does not exist
+ Turbulence generation starts ...

wetb/hawc2/tests/test_logfile.py

+'''
+Created on 18/11/2015
+
+@author: MMPE
+'''
+import unittest
+from wetb.hawc2.log_file import LogFile, is_file_open, INITIALIZATION_ERROR, \
+    INITIALIZATION, SIMULATING, DONE, SIMULATION_ERROR, GENERATING_TURBULENCE, \
+    PENDING
+import time
+from wetb.hawc2 import log_file
+import threading
+import os
+
+def simulate(file, wait):
+    with open(file, 'r') as fin:
+        lines = fin.readlines()
+    file = file + "_"
+    with open(file, 'w'):
+        pass
+    time.sleep(.1)
+    for l in lines:
+        with open(file, 'a+') as fout:
+            fout.write(l)
+        if "Turbulence generation starts" in l or "Log file output" in l:
+            time.sleep(0.2)
+        time.sleep(wait)
+
+class Test(unittest.TestCase):
+
+
+    def test_missing_logfile(self):
+        f = 'test_files/logfiles/missing.log'
+        logfile = LogFile(f, 200)
+        status = logfile.status()
+        self.assertEqual(status[0], 0)
+        self.assertEqual(status[1], log_file.MISSING)
+
+
+    def test_is_file_open(self):
+        f = 'test_files/logfiles/test.log'
+        with open(f, 'a+'):
+            self.assertTrue(is_file_open(f))
+        with open(f, 'r'):
+            self.assertTrue(is_file_open(f))
+        self.assertFalse(is_file_open(f))
+
+    def test_simulation_init_error(self):
+        f = 'test_files/logfiles/init_error.log'
+        logfile = LogFile(f, 2)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 100)
+        self.assertEqual(txt, DONE)
+        self.assertEqual(err, [' *** ERROR *** No line termination in command line            8'])
+
+    def test_init(self):
+        f = 'test_files/logfiles/init.log'
+        logfile = LogFile(f, 200)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 0)
+        self.assertEqual(txt, INITIALIZATION)
+        self.assertEqual(err, [])
+
+    def test_turbulence_generation(self):
+        f = 'test_files/logfiles/turbulence_generation.log'
+        logfile = LogFile(f, 200)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 0)
+        self.assertEqual(txt, GENERATING_TURBULENCE)
+        self.assertEqual(err, [])
+
+    def test_simulation(self):
+        f = 'test_files/logfiles/simulating.log'
+        logfile = LogFile(f, 2)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 25)
+        self.assertEqual(txt, SIMULATING)
+        self.assertEqual(err, [])
+
+
+    def test_finish(self):
+        f = 'test_files/logfiles/finish.log'
+        logfile = LogFile(f, 200)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 100)
+        self.assertEqual(txt, DONE)
+        self.assertEqual(err, [])
+
+
+    def test_simulation_error(self):
+        f = 'test_files/logfiles/simulation_error.log'
+        logfile = LogFile(f, 2)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 100)
+        self.assertEqual(txt, DONE)
+        self.assertEqual(err, [' *** ERROR *** Error opening out .dat file'])
+
+    def test_simulation_error2(self):
+        f = 'test_files/logfiles/simulation_error2.log'
+        logfile = LogFile(f, 2)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, 100)
+        self.assertEqual(txt, DONE)
+        self.assertEqual(err, ['30 x  *** ERROR *** Out of limits in user defined shear field - limit value used'])
+
+
+    def check(self, logfilename, phases, end_status):
+        logfile = LogFile(logfilename + "_", 2)
+        if os.path.isfile(logfile.filename):
+            os.remove(logfile.filename)
+        status = logfile.status()
+        t = threading.Thread(target=simulate, args=(logfilename, 0.0001))
+        t.start()
+        while status[0] >= 0 and status[1] != DONE:
+            new_status = logfile.status()
+            if new_status[1] != status[1] or new_status[0] != status[0]:
+                status = new_status
+                #print(status)
+            if status[1] in phases:
+                phases.remove(status[1])
+            time.sleep(0.01)
+        code, txt, err = logfile.status()
+        self.assertEqual(code, end_status[0])
+        self.assertEqual(txt, end_status[1])
+        self.assertEqual(err, end_status[2])
+        self.assertFalse(phases)
+        t.join()
+        os.remove(logfile.filename)
+
+
+    def test_realtime_test(self):
+        self.check('test_files/logfiles/finish.log',
+                   phases=[PENDING, INITIALIZATION, SIMULATING, DONE],
+                   end_status=(100, DONE, []))
+
+    def test_realtime_test2(self):
+        self.check('test_files/logfiles/init_error.log',
+           phases=[PENDING, INITIALIZATION, SIMULATING, DONE],
+           end_status=(100, DONE, [' *** ERROR *** No line termination in command line            8']))
+
+    def test_realtime_test_simulation_error(self):
+        self.check('test_files/logfiles/simulation_error.log',
+                   [PENDING, INITIALIZATION, SIMULATING, DONE],
+                   (100, DONE, [' *** ERROR *** Error opening out .dat file']))
+
+    def test_realtime_test_turbulence(self):
+        self.check('test_files/logfiles/finish_turbulencegeneration.log',
+                   phases=[PENDING, INITIALIZATION, GENERATING_TURBULENCE, SIMULATING, DONE],
+                   end_status=(100, DONE, []))
+
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.test_logfile']
+    unittest.main()