ascii2bin.py + test

README.md

@@ -8,6 +8,7 @@
 - [pc_file](wetb/hawc2/pc_file.py): Read PC (profile coefficient) files
 - [shear_file](wetb/hawc2/shear_file.py): Create user defined shear file
 - [at_time_file](wetb/hawc2/at_time_file.py): read at output_at_time files
+- [ascii2bin](wetb/hawc2/ascii2bin): Compress HAWC2 ascii result files to binary
 
 
 

wetb/hawc2/ascii2bin/Example.py

+'''
+Created on 06/09/2013
+
+@author: Mads M. Pedersen (mmpe@dtu.dk)
+'''
+from wetb.hawc2.ascii2bin.ascii2bin import ascii2bin
+
+
+ascii2bin(r"tests/hawc2ascii.sel", "temp_hawc2ascii.sel")
+

wetb/hawc2/ascii2bin/Run.bat

+@echo off  
+echo.
+ECHO Syntax: 
+echo ------------------------------------------------
+ECHO ascii2bin.exe {ascii_file.sel} [{bin_file}]
+echo ------------------------------------------------
+echo.
+echo if {bin_file} is omitted "test.sel" is converted into "xxx_bin.sel"
+echo.
+echo.
+cmd.exe
+echo ascii2bin.exe

wetb/hawc2/ascii2bin/__init__.py

+"""
+General Time Series Data Format - a HDF5 format for time series
+"""
+
+d = None
+d = dir()
+
+from .ascii2bin import ascii2bin, size_from_file
+
+__all__ = [m for m in set(dir()) - set(d)]

wetb/hawc2/ascii2bin/ascii2bin.py

+import sys
+import warnings
+
+warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
+
+from wetb.hawc2.ascii2bin.pandas_dat_ascii2bin import pandas_dat_ascii2bin
+
+class TextUI(object):
+    def show_message(self, m):
+        print (m)
+
+    def exec_long_task(self, text, allow_cancel, task, *args, **kwargs):
+        print (text)
+        return task(*args, **kwargs)
+
+sys.path.append(".")
+
+def size_from_file(selfilename):
+    with open(selfilename) as f:
+        info = f.readlines()[8].split()
+        scans = int(info[0])
+        no_sensors = int(info[1])
+    return (scans, no_sensors)
+
+def ascii2bin(ascii_selfilename, bin_selfilename=None, ui=TextUI()):
+
+    # Convert dat file
+    ascii_datfilename = ascii_selfilename.replace(".sel", '.dat')
+    if bin_selfilename is None:
+        bin_selfilename = ascii_selfilename[:-4] + "_bin.sel"
+
+
+    # Read, convert and write sel file
+    with open(ascii_selfilename) as f:
+        lines = f.readlines()
+
+    if "BINARY" in lines[8]:
+        ui.show_message("%s is already binary" % ascii_selfilename)
+    else:
+        #lines[1] = "  Version ID : Pydap %d.%d\n" % (version.__version__[:2])
+        lines[5] = "  Result file : %s.dat\n" % bin_selfilename[:-4]
+        lines[8] = lines[8].replace("ASCII", 'BINARY')
+        lines[-1] = lines[-1].strip() + "\n"
+        lines.append("Scale factors:\n")
+
+        scale_factors = pandas_dat_ascii2bin(ascii_datfilename, bin_selfilename.replace('.sel', '.dat'), ui)
+        for sf in scale_factors:
+            lines.append("  %.5E\n" % sf)
+        with open(bin_selfilename, 'w') as f:
+            f.writelines(lines)
+        if ui is not None:
+            ui.show_message("Finish converting %s to %s" % (ascii_selfilename, bin_selfilename))
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print ("syntax: ascii2bin ascii_sel_filename [bin_sel_filename]")
+    elif len(sys.argv) == 2:
+        ascii2bin(sys.argv[1])
+    else:
+        ascii2bin(sys.argv[1], sys.argv[2])

wetb/hawc2/ascii2bin/make_exe.py

+'''
+Created on 06/09/2013
+
+@author: Mads M. Pedersen (mmpe@dtu.dk)
+'''
+from __future__ import division, print_function, absolute_import, unicode_literals
+from build_exe.cx.build_cx_exe import NUMPY
+try: range = xrange; xrange = None
+except NameError: pass
+try: str = unicode; unicode = None
+except NameError: pass
+
+from build_exe.cx import build_cx_exe
+build_cx_exe.build_exe("ascii2bin.py", version="3.0.1", includes=["'pandas'"], modules=['email', NUMPY])

wetb/hawc2/ascii2bin/pandas_dat_ascii2bin.py

+'''
+Created on 10/01/2014
+
+@author: MMPE
+'''
+
+import numpy as np
+from pandas import read_csv
+
+
+
+def pandas_dat_ascii2bin(ascii_filename, bin_filename, ui):
+
+    df = ui.exec_long_task("Reading ascii file", False, read_csv, ascii_filename, sep=" ", skipinitialspace=True, header=None)
+
+    def compress(df, bin_filename):
+        with open(bin_filename, 'wb') as outfile:
+            scale_factors = []
+            for _, sensor in df.iteritems():
+                sf = sensor.abs().max() / 32000
+                if sf > 0:
+                    sensor /= sf
+                np.round(sensor.values).astype(np.int16).tofile(outfile)
+                scale_factors.append(sf)
+        return np.array(scale_factors)
+    #return compress(df, bin_filename)
+    return ui.exec_long_task("Compress and save as binary", False, compress, df, bin_filename)
+

wetb/hawc2/ascii2bin/readme.txt

+Syntax: 
+
+<path to asci2bin_folder>\ascii2bin.exe <ascii_file.sel> [<bin_file>]
+
+if <bin_file> is omitted test.sel is converted into test_bin.sel
+

wetb/hawc2/ascii2bin/tests/Hawc2ascii.sel

+________________________________________________________________________________________________________________________
+  Version ID : Pdap 1.2
+                                                            Time : 14:11:25
+                                                            Date : 31:03.2014
+________________________________________________________________________________________________________________________
+  Result file : C:/mmpe/programming/python/MMPE/hawc2/ascii2bin/unit_test/Hawc2ascii.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+        800     28         20.000       ASCII
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -47.50
+     3      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -47.50
+     4      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -47.50
+     5      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -50.00
+     6      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -50.00
+     7      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -50.00
+     8      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -52.50
+     9      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -52.50
+    10      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -52.50
+    11      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -47.50
+    12      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -47.50
+    13      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -47.50
+    14      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -50.00
+    15      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -50.00
+    16      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -50.00
+    17      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -52.50
+    18      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -52.50
+    19      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -52.50
+    20      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -47.50
+    21      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -47.50
+    22      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -47.50
+    23      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -50.00
+    24      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -50.00
+    25      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -50.00
+    26      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -52.50
+    27      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -52.50
+    28      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -52.50
+________________________________________________________________________________________________________________________
\ No newline at end of file

wetb/hawc2/ascii2bin/tests/Hawc2ascii_bin.sel

+________________________________________________________________________________________________________________________
+  Version ID : Pdap 1.2
+                                                            Time : 14:11:25
+                                                            Date : 31:03.2014
+________________________________________________________________________________________________________________________
+  Result file : Hawc2ascii_bin.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+        800     28         20.000       BINARY
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -47.50
+     3      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -47.50
+     4      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -47.50
+     5      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -50.00
+     6      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -50.00
+     7      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -50.00
+     8      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -52.50
+     9      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -52.50
+    10      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -52.50
+    11      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -47.50
+    12      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -47.50
+    13      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -47.50
+    14      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -50.00
+    15      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -50.00
+    16      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -50.00
+    17      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -52.50
+    18      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -52.50
+    19      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -52.50
+    20      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -47.50
+    21      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -47.50
+    22      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -47.50
+    23      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -50.00
+    24      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -50.00
+    25      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -50.00
+    26      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -52.50
+    27      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -52.50
+    28      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -52.50
+________________________________________________________________________________________________________________________
+Scale factors:
+  6.25000E-04
+  5.65541E-02
+  0.00000E+00
+  1.24203E-01
+  3.27578E-02
+  1.67890E-02
+  9.77313E-04
+  1.19489E-01
+  1.22708E-01
+  1.25926E-01
+  5.39287E-02
+  8.59994E-02
+  1.18070E-01
+  3.11474E-02
+  1.59594E-02
+  9.30853E-04
+  1.13800E-01
+  1.16866E-01
+  1.19932E-01
+  5.11688E-02
+  8.15175E-02
+  1.12097E-01
+  2.96155E-02
+  1.51702E-02
+  8.82013E-04
+  1.07819E-01
+  1.10725E-01
+  1.13630E-01

wetb/hawc2/ascii2bin/tests/Hawc2bin.sel

+________________________________________________________________________________________________________________________
+  Version ID : Pdap 1.2
+                                                            Time : 14:11:25
+                                                            Date : 31:03.2014
+________________________________________________________________________________________________________________________
+  Result file : Hawc2bin.dat
+________________________________________________________________________________________________________________________
+   Scans    Channels    Time [sec]      Format
+        800     28         20.000       BINARY
+
+  Channel   Variable Description               
+
+     1      Time                           s          Time
+     2      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -47.50
+     3      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -47.50
+     4      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -47.50
+     5      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -50.00
+     6      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -50.00
+     7      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -50.00
+     8      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,  -1.00, -52.50
+     9      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,  -1.00, -52.50
+    10      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,  -1.00, -52.50
+    11      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -47.50
+    12      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -47.50
+    13      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -47.50
+    14      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -50.00
+    15      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -50.00
+    16      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -50.00
+    17      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   0.00, -52.50
+    18      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   0.00, -52.50
+    19      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   0.00, -52.50
+    20      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -47.50
+    21      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -47.50
+    22      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -47.50
+    23      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -50.00
+    24      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -50.00
+    25      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -50.00
+    26      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    2.50,   1.00, -52.50
+    27      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos    0.00,   1.00, -52.50
+    28      WSP gl. coo.,Vy                m/s        Free wind speed Vy, gl. coo, of gl. pos   -2.50,   1.00, -52.50
+________________________________________________________________________________________________________________________
+Scale factors:
+  6.25000E-04
+  5.65541E-02
+  0.00000E+00
+  1.24203E-01
+  3.27578E-02
+  1.67890E-02
+  9.77313E-04
+  1.19489E-01
+  1.22708E-01
+  1.25926E-01
+  5.39287E-02
+  8.59994E-02
+  1.18070E-01
+  3.11474E-02
+  1.59594E-02
+  9.30853E-04
+  1.13800E-01
+  1.16866E-01
+  1.19932E-01
+  5.11688E-02
+  8.15175E-02
+  1.12097E-01
+  2.96155E-02
+  1.51702E-02
+  8.82013E-04
+  1.07819E-01
+  1.10725E-01
+  1.13630E-01

wetb/hawc2/ascii2bin/tests/test_ascii2bin.py

+'''
+Created on 29/10/2013
+
+@author: mmpe
+'''
+from wetb.hawc2 import Hawc2io
+import numpy as np
+import os
+import sys
+import unittest
+from wetb.hawc2.ascii2bin.ascii2bin import ascii2bin, size_from_file
+
+class Test(unittest.TestCase):
+
+    def setUp(self):
+        sys.path.append("../")
+
+    def testAscii2bin(self):
+
+        for f in ["Hawc2ascii_bin.sel", "Hawc2ascii_bin.dat"]:
+            if os.path.exists(f):
+                os.remove(f)
+        ascii2bin("Hawc2ascii.sel")
+
+        ascii_file = Hawc2io.ReadHawc2('Hawc2ascii')
+        bin_file = Hawc2io.ReadHawc2("Hawc2ascii_bin")
+
+        np.testing.assert_array_almost_equal(ascii_file.ReadAscii(), bin_file.ReadBinary(), 1)
+        self.assertEqual(ascii_file.ChInfo, bin_file.ChInfo)
+
+    def testAscii2bin_new_name(self):
+        for f in ["Hawc2bin.sel", "Hawc2bin.dat"]:
+            if os.path.exists(f):
+                os.remove(f)
+        ascii2bin("Hawc2ascii.sel", "Hawc2bin.sel")
+
+        ascii_file = Hawc2io.ReadHawc2('Hawc2ascii')
+        bin_file = Hawc2io.ReadHawc2("Hawc2bin")
+
+        np.testing.assert_array_almost_equal(ascii_file.ReadAscii(), bin_file.ReadBinary(), 1)
+        self.assertEqual(ascii_file.ChInfo, bin_file.ChInfo)
+
+
+    def testSizeOfFile(self):
+        self.assertEqual(size_from_file("hawc2ascii.sel"), (800, 28))
+
+
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.testName']
+    unittest.main()