windIO.py

# -*- coding: utf-8 -*-
"""
Created on Thu Apr  3 19:53:59 2014

@author: dave
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import dict
from io import open as opent
from builtins import range
from builtins import str
from builtins import int
from future import standard_library
standard_library.install_aliases()
from builtins import object

__author__ = 'David Verelst'
__license__ = 'GPL'
__version__ = '0.5'

import os
import copy
import struct
import math
from time import time
import codecs
from itertools import chain

import scipy.integrate as integrate
import numpy as np
import pandas as pd

# misc is part of prepost, which is available on the dtu wind gitlab server:
# https://gitlab.windenergy.dtu.dk/dave/prepost
from wetb.prepost import misc
# wind energy python toolbox, available on the dtu wind redmine server:
# http://vind-redmine.win.dtu.dk/projects/pythontoolbox/repository/show/fatigue_tools
from wetb.hawc2.Hawc2io import ReadHawc2
from wetb.fatigue_tools.fatigue import eq_load


class LogFile(object):
    """Check a HAWC2 log file for errors.
    """

    def __init__(self):

        # the total message list log:
        self.MsgListLog = []
        # a smaller version, just indication if there are errors:
        self.MsgListLog2 = dict()

        # specify which message to look for. The number track's the order.
        # this makes it easier to view afterwards in spreadsheet:
        # every error will have its own column

        # error messages that appear during initialisation
        self.err_init = {}
        self.err_init[' *** ERROR *** Error in com'] = len(self.err_init)
        self.err_init[' *** ERROR ***  in command '] = len(self.err_init)
        #  *** WARNING *** A comma "," is written within the command line
        self.err_init[' *** WARNING *** A comma ",'] = len(self.err_init)
        #  *** ERROR *** Not correct number of parameters
        self.err_init[' *** ERROR *** Not correct '] = len(self.err_init)
        #  *** INFO *** End of file reached
        self.err_init[' *** INFO *** End of file r'] = len(self.err_init)
        #  *** ERROR *** No line termination in command line
        self.err_init[' *** ERROR *** No line term'] = len(self.err_init)
        #  *** ERROR *** MATRIX IS NOT DEFINITE
        self.err_init[' *** ERROR *** MATRIX IS NO'] = len(self.err_init)
        #  *** ERROR *** There are unused relative
        self.err_init[' *** ERROR *** There are un'] = len(self.err_init)
        #  *** ERROR *** Error finding body based
        self.err_init[' *** ERROR *** Error findin'] = len(self.err_init)
        #  *** ERROR *** In body actions
        self.err_init[' *** ERROR *** In body acti'] = len(self.err_init)
        #  *** ERROR *** Command unknown and ignored
        self.err_init[' *** ERROR *** Command unkn'] = len(self.err_init)
        #  *** ERROR *** ERROR - More bodies than elements on main_body: tower
        self.err_init[' *** ERROR *** ERROR - More'] = len(self.err_init)
        #  *** ERROR *** The program will stop
        self.err_init[' *** ERROR *** The program '] = len(self.err_init)
        #  *** ERROR *** Unknown begin command in topologi.
        self.err_init[' *** ERROR *** Unknown begi'] = len(self.err_init)
        #  *** ERROR *** Not all needed topologi main body commands present
        self.err_init[' *** ERROR *** Not all need'] = len(self.err_init)
        #  *** ERROR ***  opening timoschenko data file
        self.err_init[' *** ERROR ***  opening tim'] = len(self.err_init)
        #  *** ERROR *** Error opening AE data file
        self.err_init[' *** ERROR *** Error openin'] = len(self.err_init)
        #  *** ERROR *** Requested blade _ae set number not found in _ae file
        self.err_init[' *** ERROR *** Requested bl'] = len(self.err_init)
        #  Error opening PC data file
        self.err_init[' Error opening PC data file'] = len(self.err_init)
        #  *** ERROR *** error reading mann turbulence
        self.err_init[' *** ERROR *** error readin'] = len(self.err_init)
        #  *** INFO *** The DLL subroutine
        self.err_init[' *** INFO *** The DLL subro'] = len(self.err_init)
        #  ** WARNING: FROM ESYS ELASTICBAR: No keyword
        self.err_init[' ** WARNING: FROM ESYS ELAS'] = len(self.err_init)
        #  *** ERROR *** DLL ./control/killtrans.dll could not be loaded - error!
        self.err_init[' *** ERROR *** DLL'] = len(self.err_init)
        # *** ERROR *** The DLL subroutine
        self.err_init[' *** ERROR *** The DLL subr'] = len(self.err_init)
        # *** ERROR *** Mann turbulence length scale must be larger than zero!
        # *** ERROR *** Mann turbulence alpha eps value must be larger than zero!
        # *** ERROR *** Mann turbulence gamma value must be larger than zero!
        self.err_init[' *** ERROR *** Mann turbule'] = len(self.err_init)

        # *** WARNING *** Shear center x location not in elastic center, set to zero
        self.err_init[' *** WARNING *** Shear cent'] = len(self.err_init)
        # Turbulence file ./xyz.bin does not exist
        self.err_init[' Turbulence file '] = len(self.err_init)
        self.err_init[' *** WARNING ***'] = len(self.err_init)
        self.err_init[' *** ERROR ***'] = len(self.err_init)
        self.err_init[' WARNING'] = len(self.err_init)
        self.err_init[' ERROR'] = len(self.err_init)

        # error messages that appear during simulation
        self.err_sim = {}
        #  *** ERROR *** Wind speed requested inside
        self.err_sim[' *** ERROR *** Wind speed r'] = len(self.err_sim)
        #  Maximum iterations exceeded at time step:
        self.err_sim[' Maximum iterations exceede'] = len(self.err_sim)
        #  Solver seems not to converge:
        self.err_sim[' Solver seems not to conver'] = len(self.err_sim)
        #  *** ERROR *** Out of x bounds:
        self.err_sim[' *** ERROR *** Out of x bou'] = len(self.err_sim)
        #  *** ERROR *** Out of limits in user defined shear field - limit value used
        self.err_sim[' *** ERROR *** Out of limit'] = len(self.err_sim)

        # TODO: error message from a non existing channel output/input
        # add more messages if required...

        self.init_cols = len(self.err_init)
        self.sim_cols = len(self.err_sim)
        self.header = None

    def readlog(self, fname, case=None, save_iter=False):
        """
        """
        # open the current log file
        with open(fname, 'r') as f:
            lines = f.readlines()

        # keep track of the messages allready found in this file
        tempLog = []
        tempLog.append(fname)
        exit_correct, found_error = False, False

        subcols_sim = 4
        subcols_init = 2
        # create empty list item for the different messages and line
        # number. Include one column for non identified messages
        for j in range(self.init_cols):
            # 2 sub-columns per message: nr, msg
            for k in range(subcols_init):
                tempLog.append('')
        for j in range(self.sim_cols):
            # 4 sub-columns per message: first, last, nr, msg
            for k in range(subcols_sim):
                tempLog.append('')
        # and two more columns at the end for messages of unknown origin
        tempLog.append('')
        tempLog.append('')

        # if there is a cases object, see how many time steps we expect
        if case is not None:
            dt = float(case['[dt_sim]'])
            time_steps = int(float(case['[time_stop]']) / dt)
            iterations = np.ndarray( (time_steps+1,3), dtype=np.float32 )
        else:
            iterations = np.ndarray( (len(lines),3), dtype=np.float32 )
            dt = False
        iterations[:,0:2] = -1
        iterations[:,2] = 0

        # keep track of the time_step number
        time_step, init_block = -1, True
        # check for messages in the current line
        # for speed: delete from message watch list if message is found
        for j, line in enumerate(lines):
            # all id's of errors are 27 characters long
            msg = line[:27]
            # remove the line terminator, this seems to take 2 characters
            # on PY2, but only one in PY3
            line = line.replace('\n', '')

            # keep track of the number of iterations
            if line[:12] == ' Global time':
                time_step += 1
                iterations[time_step,0] = float(line[14:40])
                # for PY2, new line is 2 characters, for PY3 it is one char
                iterations[time_step,1] = int(line[-6:])
                # time step is the first time stamp
                if not dt:
                    dt = float(line[15:40])
                # no need to look for messages if global time is mentioned
                continue

            elif line[:20] == ' Starting simulation':
                init_block = False

            elif init_block:
                # if string is shorter, we just get a shorter string.
                # checking presence in dict is faster compared to checking
                # the length of the string
                # first, last, nr, msg
                if msg in self.err_init:
                    # icol=0 -> fname
                    icol = subcols_init*self.err_init[msg] + 1
                    # 0: number of occurances
                    if tempLog[icol] == '':
                        tempLog[icol] = '1'
                    else:
                        tempLog[icol] = str(int(tempLog[icol]) + 1)
                    # 1: the error message itself
                    tempLog[icol+1] = line
                    found_error = True

            # find errors that can occur during simulation
            elif msg in self.err_sim:
                icol = subcols_sim*self.err_sim[msg]
                icol += subcols_init*self.init_cols + 1
                # in case stuff already goes wrong on the first time step
                if time_step == -1:
                    time_step = 0

                # 1: time step of first occurance
                if tempLog[icol]  == '':
                    tempLog[icol] = '%i' % time_step
                # 2: time step of last occurance
                tempLog[icol+1] = '%i' % time_step
                # 3: number of occurances
                if tempLog[icol+2] == '':
                    tempLog[icol+2] = '1'
                else:
                    tempLog[icol+2] = str(int(tempLog[icol+2]) + 1)
                # 4: the error message itself
                tempLog[icol+3] = line

                found_error = True
                iterations[time_step,2] = 1

            # method of last resort, we have no idea what message
            elif line[:10] == ' *** ERROR' or line[:10]==' ** WARNING':
                icol = subcols_sim*self.sim_cols
                icol += subcols_init*self.init_cols + 1
                # line number of the message
                tempLog[icol] = j
                # and message
                tempLog[icol+1] = line
                found_error = True
                # in case stuff already goes wrong on the first time step
                if time_step == -1:
                    time_step = 0
                iterations[time_step,2] = 1

        # simulation and simulation output time
        if case is not None:
            t_stop = float(case['[time_stop]'])
            duration = float(case['[duration]'])
        else:
            t_stop = -1
            duration = -1

        # see if the last line holds the sim time
        if line[:15] ==  ' Elapsed time :':
            exit_correct = True
            elapsed_time = float(line[15:-1])
            tempLog.append( elapsed_time )
        # in some cases, Elapsed time is not given, and the last message
        # might be: " Closing of external type2 DLL"
        elif line[:20] == ' Closing of external':
            exit_correct = True
            elapsed_time = iterations[time_step,0]
            tempLog.append( elapsed_time )
        elif np.allclose(iterations[time_step,0], t_stop):
            exit_correct = True
            elapsed_time = iterations[time_step,0]
            tempLog.append( elapsed_time )
        else:
            elapsed_time = -1
            tempLog.append('')

        # give the last recorded time step
        tempLog.append('%1.11f' % iterations[time_step,0])

        # simulation and simulation output time
        tempLog.append('%1.01f' % t_stop)
        tempLog.append('%1.04f' % (t_stop/elapsed_time))
        tempLog.append('%1.01f' % duration)

        # as last element, add the total number of iterations
        itertotal = np.nansum(iterations[:,1])
        tempLog.append('%i' % itertotal)

        # the delta t used for the simulation
        if dt:
            tempLog.append('%1.7f' % dt)
        else:
            tempLog.append('failed to find dt')

        # number of time steps
        tempLog.append('%i' % len(iterations) )

        # if the simulation didn't end correctly, the elapsed_time doesn't
        # exist. Add the average and maximum nr of iterations per step
        # or, if only the structural and eigen analysis is done, we have 0
        try:
            ratio = float(elapsed_time)/float(itertotal)
            tempLog.append('%1.6f' % ratio)
        except (UnboundLocalError, ZeroDivisionError, ValueError) as e:
            tempLog.append('')
        # when there are no time steps (structural analysis only)
        try:
            tempLog.append('%1.2f' % iterations[:,1].mean())
            tempLog.append('%1.2f' % iterations[:,1].max())
        except ValueError:
            tempLog.append('')
            tempLog.append('')

        # save the iterations in the results folder
        if save_iter:
            fiter = os.path.basename(fname).replace('.log', '.iter')
            fmt = ['%12.06f', '%4i', '%4i']
            if case is not None:
                fpath = os.path.join(case['[run_dir]'], case['[iter_dir]'])
                # in case it has subdirectories
                for tt in [3,2,1]:
                    tmp = os.path.sep.join(fpath.split(os.path.sep)[:-tt])
                    if not os.path.exists(tmp):
                        os.makedirs(tmp)
                if not os.path.exists(fpath):
                    os.makedirs(fpath)
                np.savetxt(fpath + fiter, iterations, fmt=fmt)
            else:
                logpath = os.path.dirname(fname)
                np.savetxt(os.path.join(logpath, fiter), iterations, fmt=fmt)

        # append the messages found in the current file to the overview log
        self.MsgListLog.append(tempLog)
        self.MsgListLog2[fname] = [found_error, exit_correct]

    def _msglistlog2csv(self, contents):
        """Write LogFile.MsgListLog to a csv file. Use LogFile._header to
        create a header.
        """
        for k in self.MsgListLog:
            for n in k:
                contents = contents + str(n) + ';'
            # at the end of each line, new line symbol
            contents = contents + '\n'
        return contents

    def csv2df(self, fname):
        """Read a csv log file analysis and convert to a pandas.DataFrame
        """
        colnames, min_itemsize, dtypes = self.headers4df()
        df = pd.read_csv(fname, header=0, names=colnames, sep=';', )
        for col, dtype in dtypes.items():
            df[col] = df[col].astype(dtype)
            # replace nan with empty for str columns
            if dtype == str:
                df[col] = df[col].str.replace('nan', '')
        return df

    def _header(self):
        """Header for log analysis csv file
        """

        # write the results in a file, start with a header
        contents = 'file name;' + 'nr;msg;'*(self.init_cols)
        contents += 'first_tstep;last_tstep;nr;msg;'*(self.sim_cols)
        contents += 'lnr;msg;'
        # and add headers for elapsed time, nr of iterations, and sec/iteration
        contents += 'Elapsted time;last time step;Simulation time;'
        contents += 'real sim time;Sim output time;'
        contents += 'total iterations;dt;nr time steps;'
        contents += 'seconds/iteration;average iterations/time step;'
        contents += 'maximum iterations/time step;\n'

        return contents

    def headers4df(self):
        """Create header and a minimum itemsize for string columns when
        converting a Log check analysis to a pandas.DataFrame

        Returns
        -------

        header : list
            List of column names as generated by WindIO.LogFile._header

        min_itemsize : dict
            Dictionary with column names as keys, and the minimum string lenght
            as values.

        dtypes : dict
            Dictionary with column names as keys, and data types as values
        """
        chain_iter = chain.from_iterable

        colnames = ['file_name']
        colnames.extend(list(chain_iter(('nr_%i' % i, 'msg_%i' % i)
                      for i in range(31))) )

        gr = ('first_tstep_%i', 'last_step_%i', 'nr_%i', 'msg_%i')
        colnames.extend(list(chain_iter( (k % i for k in gr)
                           for i in range(100,105,1))) )
        colnames.extend(['nr_extra', 'msg_extra'])
        colnames.extend(['elapsted_time',
                       'last_time_step',
                       'simulation_time',
                       'real_sim_time',
                       'sim_output_time',
                       'total_iterations',
                       'dt',
                       'nr_time_steps',
                       'seconds_p_iteration',
                       'mean_iters_p_time_step',
                       'max_iters_p_time_step',
                       'sim_id'])
        dtypes = {}

        # str and float datatypes for
        msg_cols = ['msg_%i' % i for i in range(30)]
        msg_cols.extend(['msg_%i' % i for i in range(100,105,1)])
        dtypes.update({k:str for k in msg_cols})
        # make the message/str columns long enough
        min_itemsize = {'msg_%i' % i : 100 for i in range(30)}

        # column names holding the number of occurances of messages
        nr_cols = ['nr_%i' % i for i in range(30)]
        nr_cols.extend(['nr_%i' % i for i in range(100,105,1)])
        # other float values
        nr_cols.extend(['elapsted_time', 'total_iterations'])
        # NaN only exists in float arrays, not integers (NumPy limitation)
        # so use float instead of int
        dtypes.update({k:np.float64 for k in nr_cols})

        return colnames, min_itemsize, dtypes


class LoadResults(ReadHawc2):
    """Read a HAWC2 result data file

    Usage:
    obj = LoadResults(file_path, file_name)

    This class is called like a function:
    HawcResultData() will read the specified file upon object initialization.

    Available output:
    obj.sig[timeStep,channel]   : complete result file in a numpy array
    obj.ch_details[channel,(0=ID; 1=units; 2=description)] : np.array
    obj.error_msg: is 'none' if everything went OK, otherwise it holds the
    error

    The ch_dict key/values pairs are structured differently for different
        type of channels. Currently supported channels are:

        For forcevec, momentvec, state commands:
            key:
                coord-bodyname-pos-sensortype-component
                global-tower-node-002-forcevec-z
                local-blade1-node-005-momentvec-z
                hub1-blade1-elem-011-zrel-1.00-state pos-z
            value:
                ch_dict[tag]['coord']
                ch_dict[tag]['bodyname']
                ch_dict[tag]['pos'] = pos
                ch_dict[tag]['sensortype']
                ch_dict[tag]['component']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the DLL's this is:
            key:
                DLL-dll_name-io-io_nr
                DLL-yaw_control-outvec-3
                DLL-yaw_control-inpvec-1
            value:
                ch_dict[tag]['dll_name']
                ch_dict[tag]['io']
                ch_dict[tag]['io_nr']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the bearings this is:
            key:
                bearing-bearing_name-output_type-units
                bearing-shaft_nacelle-angle_speed-rpm
            value:
                ch_dict[tag]['bearing_name']
                ch_dict[tag]['output_type']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']

    """
    # ch_df columns, these are created by LoadResults._unified_channel_names
    cols = set(['bearing_name', 'sensortag', 'bodyname', 'chi', 'component',
                'pos', 'coord', 'sensortype', 'radius', 'blade_nr', 'units',
                'output_type', 'io_nr', 'io', 'dll', 'azimuth', 'flap_nr',
                'direction'])

    # start with reading the .sel file, containing the info regarding
    # how to read the binary file and the channel information
    def __init__(self, file_path, file_name, debug=False, usecols=None,
                 readdata=True):

        self.debug = debug

        # timer in debug mode
        if self.debug:
            start = time()

        self.file_path = file_path
        # remove .log, .dat, .sel extensions who might be accedental left
        if file_name[-4:] in ['.htc', '.sel', '.dat', '.log']:
            file_name = file_name[:-4]
        # FIXME: since HAWC2 will always have lower case output files, convert
        # any wrongly used upper case letters to lower case here
        self.file_name = file_name
        FileName = os.path.join(self.file_path, self.file_name)

        ReadOnly = 0 if readdata else 1
        super(LoadResults, self).__init__(FileName, ReadOnly=ReadOnly)
        self.FileType = self.FileFormat[6:]
        self.N = int(self.NrSc)
        self.Nch = int(self.NrCh)
        self.ch_details = np.ndarray(shape=(self.Nch, 3), dtype='<U100')
        for ic in range(self.Nch):
            self.ch_details[ic, 0] = self.ChInfo[0][ic]
            self.ch_details[ic, 1] = self.ChInfo[1][ic]
            self.ch_details[ic, 2] = self.ChInfo[2][ic]

        ChVec = [] if usecols is None else usecols

        self._unified_channel_names()
        if readdata:
            self.sig = super(LoadResults, self).__call__(ChVec=ChVec)

        if self.debug:
            stop = time() - start
            print('time to load HAWC2 file:', stop, 's')


    def reformat_sig_details(self):
        """Change HAWC2 output description of the channels short descriptive
        strings, usable in plots

        obj.ch_details[channel,(0=ID; 1=units; 2=description)] : np.array
        """

        # CONFIGURATION: mappings between HAWC2 and short good output:
        change_list = []
        change_list.append( ['original', 'new improved'] )

#        change_list.append( ['Mx coo: hub1','blade1 root bending: flap'] )
#        change_list.append( ['My coo: hub1','blade1 root bending: edge'] )
#        change_list.append( ['Mz coo: hub1','blade1 root bending: torsion'] )
#
#        change_list.append( ['Mx coo: hub2','blade2 root bending: flap'] )
#        change_list.append( ['My coo: hub2','blade2 root bending: edge'] )
#        change_list.append( ['Mz coo: hub2','blade2 root bending: torsion'] )
#
#        change_list.append( ['Mx coo: hub3','blade3 root bending: flap'] )
#        change_list.append( ['My coo: hub3','blade3 root bending: edge'] )
#        change_list.append( ['Mz coo: hub3','blade3 root bending: torsion'] )

        change_list.append(['Mx coo: blade1', 'blade1 flap'])
        change_list.append(['My coo: blade1', 'blade1 edge'])
        change_list.append(['Mz coo: blade1', 'blade1 torsion'])

        change_list.append(['Mx coo: blade2', 'blade2 flap'])
        change_list.append(['My coo: blade2', 'blade2 edge'])
        change_list.append(['Mz coo: blade2', 'blade2 torsion'])

        change_list.append(['Mx coo: blade3', 'blade3 flap'])
        change_list.append(['My coo: blade3', 'blade3 edeg'])
        change_list.append(['Mz coo: blade3', 'blade3 torsion'])

        change_list.append(['Mx coo: hub1', 'blade1 out-of-plane'])
        change_list.append(['My coo: hub1', 'blade1 in-plane'])
        change_list.append(['Mz coo: hub1', 'blade1 torsion'])

        change_list.append(['Mx coo: hub2', 'blade2 out-of-plane'])
        change_list.append(['My coo: hub2', 'blade2 in-plane'])
        change_list.append(['Mz coo: hub2', 'blade2 torsion'])

        change_list.append(['Mx coo: hub3', 'blade3 out-of-plane'])
        change_list.append(['My coo: hub3', 'blade3 in-plane'])
        change_list.append(['Mz coo: hub3', 'blade3 torsion'])
        # this one will create a false positive for tower node nr1
        change_list.append(['Mx coo: tower', 'tower top momemt FA'])
        change_list.append(['My coo: tower', 'tower top momemt SS'])
        change_list.append(['Mz coo: tower', 'yaw-moment'])

        change_list.append(['Mx coo: chasis', 'chasis momemt FA'])
        change_list.append(['My coo: chasis', 'yaw-moment chasis'])
        change_list.append(['Mz coo: chasis', 'chasis moment SS'])

        change_list.append(['DLL inp  2:  2', 'tower clearance'])

        self.ch_details_new = np.ndarray(shape=(self.Nch, 3), dtype='<U100')

        # approach: look for a specific description and change it.
        # This approach is slow, but will not fail if the channel numbers change
        # over different simulations
        for ch in range(self.Nch):
            # the change_list will always be slower, so this loop will be
            # inside the bigger loop of all channels
            self.ch_details_new[ch, :] = self.ch_details[ch, :]
            for k in range(len(change_list)):
                if change_list[k][0] == self.ch_details[ch, 0]:
                    self.ch_details_new[ch, 0] = change_list[k][1]
                    # channel description should be unique, so delete current
                    # entry and stop looking in the change list
                    del change_list[k]
                    break

#        self.ch_details_new = ch_details_new

    def _unified_channel_names(self):
        """
        Make certain channels independent from their index.

        The unified channel dictionary ch_dict holds consequently named
        channels as the key, and the all information is stored in the value
        as another dictionary.

        The ch_dict key/values pairs are structured differently for different
        type of channels. Currently supported channels are:

        For forcevec, momentvec, state commands:
            node numbers start with 0 at the root
            element numbers start with 1 at the root
            key:
                coord-bodyname-pos-sensortype-component
                global-tower-node-002-forcevec-z
                local-blade1-node-005-momentvec-z
                hub1-blade1-elem-011-zrel-1.00-state pos-z
            value:
                ch_dict[tag]['coord']
                ch_dict[tag]['bodyname']
                ch_dict[tag]['pos']
                ch_dict[tag]['sensortype']
                ch_dict[tag]['component']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the DLL's this is:
            key:
                DLL-dll_name-io-io_nr
                DLL-yaw_control-outvec-3
                DLL-yaw_control-inpvec-1
            value:
                ch_dict[tag]['dll_name']
                ch_dict[tag]['io']
                ch_dict[tag]['io_nr']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the bearings this is:
            key:
                bearing-bearing_name-output_type-units
                bearing-shaft_nacelle-angle_speed-rpm
            value:
                ch_dict[tag]['bearing_name']
                ch_dict[tag]['output_type']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']

        For many of the aero sensors:
            'Cl', 'Cd', 'Alfa', 'Vrel'
            key:
                sensortype-blade_nr-pos
                Cl-1-0.01
            value:
                ch_dict[tag]['sensortype']
                ch_dict[tag]['blade_nr']
                ch_dict[tag]['pos']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']
        """
        # save them in a dictionary, use the new coherent naming structure
        # as the key, and as value again a dict that hols all the different
        # classifications: (chi, channel nr), (coord, coord), ...
        self.ch_dict = dict()

        # some channel ID's are unique, use them
        ch_unique = set(['Omega', 'Ae rot. torque', 'Ae rot. power',
                         'Ae rot. thrust', 'Time', 'Azi  1'])
        ch_aero = set(['Cl', 'Cd', 'Alfa', 'Vrel', 'Tors_e', 'Alfa'])
        ch_aerogrid = set(['a_grid', 'am_grid'])

        # also safe as df
#        cols = set(['bearing_name', 'sensortag', 'bodyname', 'chi',
#                    'component', 'pos', 'coord', 'sensortype', 'radius',
#                    'blade_nr', 'units', 'output_type', 'io_nr', 'io', 'dll',
#                    'azimuth', 'flap_nr'])
        df_dict = {col: [] for col in self.cols}
        df_dict['unique_ch_name'] = []

        # scan through all channels and see which can be converted
        # to sensible unified name
        for ch in range(self.Nch):
            items = self.ch_details[ch, 2].split(' ')
            # remove empty values in the list
            items = misc.remove_items(items, '')

            dll = False

            # be carefull, identify only on the starting characters, because
            # the signal tag can hold random text that in some cases might
            # trigger a false positive

            # -----------------------------------------------------------------
            # check for all the unique channel descriptions
            if self.ch_details[ch,0].strip() in ch_unique:
                tag = self.ch_details[ch, 0].strip()
                channelinfo = {}
                channelinfo['units'] = self.ch_details[ch, 1]
                channelinfo['sensortag'] = self.ch_details[ch, 2]
                channelinfo['chi'] = ch

            # -----------------------------------------------------------------
            # or in the long description:
            #    0          1        2      3  4    5     6 and up
            # MomentMz Mbdy:blade nodenr:   5 coo: blade  TAG TEXT
            elif self.ch_details[ch, 2].startswith('MomentM'):
                coord = items[5]
                bodyname = items[1].replace('Mbdy:', '')
                # set nodenr to sortable way, include leading zeros
                # node numbers start with 0 at the root
                nodenr = '%03i' % int(items[3])
                # skip the attached the component
                # sensortype = items[0][:-2]
                # or give the sensor type the same name as in HAWC2
                sensortype = 'momentvec'
                component = items[0][-1:len(items[0])]
                # the tag only exists if defined
                if len(items) > 6:
                    sensortag = ' '.join(items[6:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'node-%s' % nodenr
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            #   0    1      2        3       4  5     6     7 and up
            # Force  Fx Mbdy:blade nodenr:   2 coo: blade  TAG TEXT
            elif self.ch_details[ch, 2].startswith('Force'):
                coord = items[6]
                bodyname = items[2].replace('Mbdy:', '')
                nodenr = '%03i' % int(items[4])
                # skipe the attached the component
                # sensortype = items[0]
                # or give the sensor type the same name as in HAWC2
                sensortype = 'forcevec'
                component = items[1][1]
                if len(items) > 7:
                    sensortag = ' '.join(items[7:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'node-%s' % nodenr
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            #   0    1  2      3       4      5   6         7    8
            # State pos x  Mbdy:blade E-nr:   1 Z-rel:0.00 coo: blade
            #   0           1     2    3        4    5   6         7     8     9+
            # State_rot proj_ang tx Mbdy:bname E-nr: 1 Z-rel:0.00 coo: cname  label
            # State_rot omegadot tz Mbdy:bname E-nr: 1 Z-rel:1.00 coo: cname  label
            elif self.ch_details[ch,2].startswith('State'):
#                 or self.ch_details[ch,0].startswith('euler') \
#                 or self.ch_details[ch,0].startswith('ax') \
#                 or self.ch_details[ch,0].startswith('omega') \
#                 or self.ch_details[ch,0].startswith('proj'):
                coord = items[8]
                bodyname = items[3].replace('Mbdy:', '')
                # element numbers start with 1 at the root
                elementnr = '%03i' % int(items[5])
                zrel = '%04.2f' % float(items[6].replace('Z-rel:', ''))
                # skip the attached the component
                #sensortype = ''.join(items[0:2])
                # or give the sensor type the same name as in HAWC2
                tmp = self.ch_details[ch, 0].split(' ')
                sensortype = tmp[0]
                if sensortype.startswith('State'):
                    sensortype += ' ' + tmp[1]
                component = items[2]
                if len(items) > 8:
                    sensortag = ' '.join(items[9:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'elem-%s-zrel-%s' % (elementnr, zrel)
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            # DLL CONTROL I/O
            # there are two scenario's on how the channel description is formed
            # the channel id is always the same though
            # id for all three cases:
            #          DLL out  1:  3
            #          DLL inp  2:  3
            # description case 1 ("dll type2_dll b2h2 inpvec 30" in htc output)
            #               0         1    2   3     4+
            #          yaw_control outvec  3  yaw_c input reference angle
            # description case 2 ("dll inpvec 2 1" in htc output):
            #           0  1 2     3  4  5  6+
            #          DLL : 2 inpvec :  4  mgen hss
            # description case 3
            #           0         1     2       4
            #          hawc_dll :echo outvec :  1
            elif self.ch_details[ch, 0].startswith('DLL'):
                # case 3
                if items[1][0] == ':echo':
                    # hawc_dll named case (case 3) is polluted with colons
                    items = self.ch_details[ch,2].replace(':', '')
                    items = items.split(' ')
                    items = misc.remove_items(items, '')
                    dll = items[1]
                    io = items[2]
                    io_nr = items[3]
                    tag = 'DLL-%s-%s-%s' % (dll, io, io_nr)
                    sensortag = ''
                # case 2: no reference to dll name
                elif self.ch_details[ch,2].startswith('DLL'):
                    dll = items[2]
                    io = items[3]
                    io_nr = items[5]
                    sensortag = ' '.join(items[6:])
                    # and tag it
                    tag = 'DLL-%s-%s-%s' % (dll,io,io_nr)
                # case 1: type2 dll name is given
                else:
                    dll = items[0]
                    io = items[1]
                    io_nr = items[2]
                    sensortag = ' '.join(items[3:])
                    tag = 'DLL-%s-%s-%s' % (dll, io, io_nr)

                # save all info in the dict
                channelinfo = {}
                channelinfo['dll'] = dll
                channelinfo['io'] = io
                channelinfo['io_nr'] = io_nr
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            # BEARING OUTPUS
            # bea1 angle_speed       rpm      shaft_nacelle angle speed
            elif self.ch_details[ch, 0].startswith('bea'):
                output_type = self.ch_details[ch, 0].split(' ')[1]
                bearing_name = items[0]
                units = self.ch_details[ch, 1]
                # there is no label option for the bearing output

                # and tag it
                tag = 'bearing-%s-%s-%s' % (bearing_name, output_type, units)
                # save all info in the dict
                channelinfo = {}
                channelinfo['bearing_name'] = bearing_name
                channelinfo['output_type'] = output_type
                channelinfo['units'] = units
                channelinfo['chi'] = ch

            # -----------------------------------------------------------------
            # AERO CL, CD, CM, VREL, ALFA, LIFT, DRAG, etc
            # Cl, R=  0.5     deg      Cl of blade  1 at radius   0.49
            # Azi  1          deg      Azimuth of blade  1
            elif self.ch_details[ch, 0].split(',')[0] in ch_aero:
                dscr_list = self.ch_details[ch, 2].split(' ')
                dscr_list = misc.remove_items(dscr_list, '')

                sensortype = self.ch_details[ch, 0].split(',')[0]
                radius = dscr_list[-1]
                # is this always valid?
                blade_nr = self.ch_details[ch, 2].split('blade  ')[1][0]
                # sometimes the units for aero sensors are wrong!
                units = self.ch_details[ch, 1]
                # there is no label option

                # and tag it
                tag = '%s-%s-%s' % (sensortype, blade_nr, radius)
                # save all info in the dict
                channelinfo = {}
                channelinfo['sensortype'] = sensortype
                channelinfo['radius'] = float(radius)
                channelinfo['blade_nr'] = int(blade_nr)
                channelinfo['units'] = units
                channelinfo['chi'] = ch

            # -----------------------------------------------------------------
            # for the induction grid over the rotor
            # a_grid, azi    0.00 r   1.74
            elif self.ch_details[ch, 0].split(',')[0] in ch_aerogrid:
                items = self.ch_details[ch, 0].split(',')
                sensortype = items[0]
                items2 = items[1].split(' ')
                items2 = misc.remove_items(items2, '')
                azi = items2[1]
                radius = items2[3]
                units = self.ch_details[ch, 1]
                # and tag it
                tag = '%s-azi-%s-r-%s' % (sensortype,azi,radius)
                # save all info in the dict
                channelinfo = {}
                channelinfo['sensortype'] = sensortype
                channelinfo['radius'] = float(radius)
                channelinfo['azimuth'] = float(azi)
                channelinfo['units'] = units
                channelinfo['chi'] = ch

            # -----------------------------------------------------------------
            # INDUCTION AT THE BLADE
            # 0: Induc. Vz, rpco, R=  1.4
            # 1: m/s
            # 2: Induced wsp Vz of blade  1 at radius   1.37, RP. coo.
# Induc. Vx, locco, R=  1.4 // Induced wsp Vx of blade  1 at radius   1.37, local ae coo.
# Induc. Vy, blco, R=  1.4 // Induced wsp Vy of blade  1 at radius   1.37, local bl coo.
# Induc. Vz, glco, R=  1.4 // Induced wsp Vz of blade  1 at radius   1.37, global coo.
# Induc. Vx, rpco, R=  8.4 // Induced wsp Vx of blade  1 at radius   8.43, RP. coo.
            elif self.ch_details[ch, 0].strip()[:5] == 'Induc':
                items = self.ch_details[ch, 2].split(' ')
                items = misc.remove_items(items, '')
                blade_nr = int(items[5])
                radius = float(items[8].replace(',', ''))
                items = self.ch_details[ch, 0].split(',')
                coord = items[1].strip()
                component = items[0][-2:]
                units = self.ch_details[ch, 1]
                # and tag it
                rpl = (coord, blade_nr, component, radius)
                tag = 'induc-%s-blade-%1i-%s-r-%03.02f' % rpl
                # save all info in the dict
                channelinfo = {}
                channelinfo['blade_nr'] = blade_nr
                channelinfo['sensortype'] = 'induction'
                channelinfo['radius'] = radius
                channelinfo['coord'] = coord
                channelinfo['component'] = component
                channelinfo['units'] = units
                channelinfo['chi'] = ch

            # TODO: wind speed
            # some spaces have been trimmed here
            # WSP gl. coo.,Vy          m/s
            # // Free wind speed Vy, gl. coo, of gl. pos   0.00,  0.00,  -2.31
            # WSP gl. coo.,Vdir_hor          deg