windIO.py

# -*- coding: utf-8 -*-
"""
Created on Thu Apr  3 19:53:59 2014

@author: dave
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import dict
from io import open as opent
from builtins import range
from builtins import str
from builtins import int
from future import standard_library
standard_library.install_aliases()
from builtins import object

__author__ = 'David Verelst'
__license__ = 'GPL'
__version__ = '0.5'

import os
import copy
import struct
import math
from time import time
import codecs
from itertools import chain

import numpy as np
import scipy as sp
import scipy.integrate as integrate
import pandas as pd

# misc is part of prepost, which is available on the dtu wind gitlab server:
# https://gitlab.windenergy.dtu.dk/dave/prepost
from wetb.prepost import misc
# wind energy python toolbox, available on the dtu wind redmine server:
# http://vind-redmine.win.dtu.dk/projects/pythontoolbox/repository/show/fatigue_tools
from wetb.hawc2.Hawc2io import ReadHawc2
from wetb.fatigue_tools.fatigue import eq_load


class LogFile(object):
    """Check a HAWC2 log file for errors.
    """

    def __init__(self):

        # the total message list log:
        self.MsgListLog = []
        # a smaller version, just indication if there are errors:
        self.MsgListLog2 = dict()

        # specify which message to look for. The number track's the order.
        # this makes it easier to view afterwards in spreadsheet:
        # every error will have its own column

        # error messages that appear during initialisation
        self.err_init = {}
        self.err_init[' *** ERROR *** Error in com'] = len(self.err_init)
        self.err_init[' *** ERROR ***  in command '] = len(self.err_init)
        #  *** WARNING *** A comma "," is written within the command line
        self.err_init[' *** WARNING *** A comma ",'] = len(self.err_init)
        #  *** ERROR *** Not correct number of parameters
        self.err_init[' *** ERROR *** Not correct '] = len(self.err_init)
        #  *** INFO *** End of file reached
        self.err_init[' *** INFO *** End of file r'] = len(self.err_init)
        #  *** ERROR *** No line termination in command line
        self.err_init[' *** ERROR *** No line term'] = len(self.err_init)
        #  *** ERROR *** MATRIX IS NOT DEFINITE
        self.err_init[' *** ERROR *** MATRIX IS NO'] = len(self.err_init)
        #  *** ERROR *** There are unused relative
        self.err_init[' *** ERROR *** There are un'] = len(self.err_init)
        #  *** ERROR *** Error finding body based
        self.err_init[' *** ERROR *** Error findin'] = len(self.err_init)
        #  *** ERROR *** In body actions
        self.err_init[' *** ERROR *** In body acti'] = len(self.err_init)
        #  *** ERROR *** Command unknown and ignored
        self.err_init[' *** ERROR *** Command unkn'] = len(self.err_init)
        #  *** ERROR *** ERROR - More bodies than elements on main_body: tower
        self.err_init[' *** ERROR *** ERROR - More'] = len(self.err_init)
        #  *** ERROR *** The program will stop
        self.err_init[' *** ERROR *** The program '] = len(self.err_init)
        #  *** ERROR *** Unknown begin command in topologi.
        self.err_init[' *** ERROR *** Unknown begi'] = len(self.err_init)
        #  *** ERROR *** Not all needed topologi main body commands present
        self.err_init[' *** ERROR *** Not all need'] = len(self.err_init)
        #  *** ERROR ***  opening timoschenko data file
        self.err_init[' *** ERROR ***  opening tim'] = len(self.err_init)
        #  *** ERROR *** Error opening AE data file
        self.err_init[' *** ERROR *** Error openin'] = len(self.err_init)
        #  *** ERROR *** Requested blade _ae set number not found in _ae file
        self.err_init[' *** ERROR *** Requested bl'] = len(self.err_init)
        #  Error opening PC data file
        self.err_init[' Error opening PC data file'] = len(self.err_init)
        #  *** ERROR *** error reading mann turbulence
        self.err_init[' *** ERROR *** error readin'] = len(self.err_init)
        #  *** INFO *** The DLL subroutine
        self.err_init[' *** INFO *** The DLL subro'] = len(self.err_init)
        #  ** WARNING: FROM ESYS ELASTICBAR: No keyword
        self.err_init[' ** WARNING: FROM ESYS ELAS'] = len(self.err_init)
        #  *** ERROR *** DLL ./control/killtrans.dll could not be loaded - error!
        self.err_init[' *** ERROR *** DLL'] = len(self.err_init)
        # *** ERROR *** The DLL subroutine
        self.err_init[' *** ERROR *** The DLL subr'] = len(self.err_init)
        # *** ERROR *** Mann turbulence length scale must be larger than zero!
        # *** ERROR *** Mann turbulence alpha eps value must be larger than zero!
        # *** ERROR *** Mann turbulence gamma value must be larger than zero!
        self.err_init[' *** ERROR *** Mann turbule'] = len(self.err_init)

        # *** WARNING *** Shear center x location not in elastic center, set to zero
        self.err_init[' *** WARNING *** Shear cent'] = len(self.err_init)
        # Turbulence file ./xyz.bin does not exist
        self.err_init[' Turbulence file '] = len(self.err_init)
        self.err_init[' *** WARNING ***'] = len(self.err_init)
        self.err_init[' *** ERROR ***'] = len(self.err_init)
        self.err_init[' WARNING'] = len(self.err_init)
        self.err_init[' ERROR'] = len(self.err_init)

        # error messages that appear during simulation
        self.err_sim = {}
        #  *** ERROR *** Wind speed requested inside
        self.err_sim[' *** ERROR *** Wind speed r'] = len(self.err_sim)
        #  Maximum iterations exceeded at time step:
        self.err_sim[' Maximum iterations exceede'] = len(self.err_sim)
        #  Solver seems not to converge:
        self.err_sim[' Solver seems not to conver'] = len(self.err_sim)
        #  *** ERROR *** Out of x bounds:
        self.err_sim[' *** ERROR *** Out of x bou'] = len(self.err_sim)
        #  *** ERROR *** Out of limits in user defined shear field - limit value used
        self.err_sim[' *** ERROR *** Out of limit'] = len(self.err_sim)

        # TODO: error message from a non existing channel output/input
        # add more messages if required...

        self.init_cols = len(self.err_init)
        self.sim_cols = len(self.err_sim)
        self.header = None

    def readlog(self, fname, case=None, save_iter=False):
        """
        """
        # open the current log file
        with open(fname, 'r') as f:
            lines = f.readlines()

        # keep track of the messages allready found in this file
        tempLog = []
        tempLog.append(fname)
        exit_correct, found_error = False, False

        subcols_sim = 4
        subcols_init = 2
        # create empty list item for the different messages and line
        # number. Include one column for non identified messages
        for j in range(self.init_cols):
            # 2 sub-columns per message: nr, msg
            for k in range(subcols_init):
                tempLog.append('')
        for j in range(self.sim_cols):
            # 4 sub-columns per message: first, last, nr, msg
            for k in range(subcols_sim):
                tempLog.append('')
        # and two more columns at the end for messages of unknown origin
        tempLog.append('')
        tempLog.append('')

        # if there is a cases object, see how many time steps we expect
        if case is not None:
            dt = float(case['[dt_sim]'])
            time_steps = int(float(case['[time_stop]']) / dt)
            iterations = np.ndarray( (time_steps+1,3), dtype=np.float32 )
        else:
            iterations = np.ndarray( (len(lines),3), dtype=np.float32 )
            dt = False
        iterations[:,0:2] = -1
        iterations[:,2] = 0

        # keep track of the time_step number
        time_step, init_block = -1, True
        # check for messages in the current line
        # for speed: delete from message watch list if message is found
        for j, line in enumerate(lines):
            # all id's of errors are 27 characters long
            msg = line[:27]
            # remove the line terminator, this seems to take 2 characters
            # on PY2, but only one in PY3
            line = line.replace('\n', '')

            # keep track of the number of iterations
            if line[:12] == ' Global time':
                time_step += 1
                iterations[time_step,0] = float(line[14:40])
                # for PY2, new line is 2 characters, for PY3 it is one char
                iterations[time_step,1] = int(line[-6:])
                # time step is the first time stamp
                if not dt:
                    dt = float(line[15:40])
                # no need to look for messages if global time is mentioned
                continue

            elif line[:4] == ' kfw':
                pass
            # Global time =    17.7800000000000      Iter =            2
            # kfw  0.861664060457402
            #  nearwake iterations          17

            # computed relaxation factor  0.300000000000000


            elif line[:20] == ' Starting simulation':
                init_block = False

            elif init_block:
                # if string is shorter, we just get a shorter string.
                # checking presence in dict is faster compared to checking
                # the length of the string
                # first, last, nr, msg
                if msg in self.err_init:
                    # icol=0 -> fname
                    icol = subcols_init*self.err_init[msg] + 1
                    # 0: number of occurances
                    if tempLog[icol] == '':
                        tempLog[icol] = '1'
                    else:
                        tempLog[icol] = str(int(tempLog[icol]) + 1)
                    # 1: the error message itself
                    tempLog[icol+1] = line
                    found_error = True

            # find errors that can occur during simulation
            elif msg in self.err_sim:
                icol = subcols_sim*self.err_sim[msg]
                icol += subcols_init*self.init_cols + 1
                # in case stuff already goes wrong on the first time step
                if time_step == -1:
                    time_step = 0

                # 1: time step of first occurance
                if tempLog[icol]  == '':
                    tempLog[icol] = '%i' % time_step
                # 2: time step of last occurance
                tempLog[icol+1] = '%i' % time_step
                # 3: number of occurances
                if tempLog[icol+2] == '':
                    tempLog[icol+2] = '1'
                else:
                    tempLog[icol+2] = str(int(tempLog[icol+2]) + 1)
                # 4: the error message itself
                tempLog[icol+3] = line

                found_error = True
                iterations[time_step,2] = 1

            # method of last resort, we have no idea what message
            elif line[:10] == ' *** ERROR' or line[:10]==' ** WARNING':
                icol = subcols_sim*self.sim_cols
                icol += subcols_init*self.init_cols + 1
                # line number of the message
                tempLog[icol] = j
                # and message
                tempLog[icol+1] = line
                found_error = True
                # in case stuff already goes wrong on the first time step
                if time_step == -1:
                    time_step = 0
                iterations[time_step,2] = 1

        # simulation and simulation output time
        if case is not None:
            t_stop = float(case['[time_stop]'])
            duration = float(case['[duration]'])
        else:
            t_stop = -1
            duration = -1

        # see if the last line holds the sim time
        if line[:15] ==  ' Elapsed time :':
            exit_correct = True
            elapsed_time = float(line[15:-1])
            tempLog.append( elapsed_time )
        # in some cases, Elapsed time is not given, and the last message
        # might be: " Closing of external type2 DLL"
        elif line[:20] == ' Closing of external':
            exit_correct = True
            elapsed_time = iterations[time_step,0]
            tempLog.append( elapsed_time )
        elif np.allclose(iterations[time_step,0], t_stop):
            exit_correct = True
            elapsed_time = iterations[time_step,0]
            tempLog.append( elapsed_time )
        else:
            elapsed_time = -1
            tempLog.append('')

        # give the last recorded time step
        tempLog.append('%1.11f' % iterations[time_step,0])

        # simulation and simulation output time
        tempLog.append('%1.01f' % t_stop)
        tempLog.append('%1.04f' % (t_stop/elapsed_time))
        tempLog.append('%1.01f' % duration)

        # as last element, add the total number of iterations
        itertotal = np.nansum(iterations[:,1])
        tempLog.append('%i' % itertotal)

        # the delta t used for the simulation
        if dt:
            tempLog.append('%1.7f' % dt)
        else:
            tempLog.append('failed to find dt')

        # number of time steps
        tempLog.append('%i' % len(iterations) )

        # if the simulation didn't end correctly, the elapsed_time doesn't
        # exist. Add the average and maximum nr of iterations per step
        # or, if only the structural and eigen analysis is done, we have 0
        try:
            ratio = float(elapsed_time)/float(itertotal)
            tempLog.append('%1.6f' % ratio)
        except (UnboundLocalError, ZeroDivisionError, ValueError) as e:
            tempLog.append('')
        # when there are no time steps (structural analysis only)
        try:
            tempLog.append('%1.2f' % iterations[:,1].mean())
            tempLog.append('%1.2f' % iterations[:,1].max())
        except ValueError:
            tempLog.append('')
            tempLog.append('')

        # save the iterations in the results folder
        if save_iter:
            fiter = os.path.basename(fname).replace('.log', '.iter')
            fmt = ['%12.06f', '%4i', '%4i']
            if case is not None:
                fpath = os.path.join(case['[run_dir]'], case['[iter_dir]'])
                # in case it has subdirectories
                for tt in [3,2,1]:
                    tmp = os.path.sep.join(fpath.split(os.path.sep)[:-tt])
                    if not os.path.exists(tmp):
                        os.makedirs(tmp)
                if not os.path.exists(fpath):
                    os.makedirs(fpath)
                np.savetxt(fpath + fiter, iterations, fmt=fmt)
            else:
                logpath = os.path.dirname(fname)
                np.savetxt(os.path.join(logpath, fiter), iterations, fmt=fmt)

        # append the messages found in the current file to the overview log
        self.MsgListLog.append(tempLog)
        self.MsgListLog2[fname] = [found_error, exit_correct]

    def _msglistlog2csv(self, contents):
        """Write LogFile.MsgListLog to a csv file. Use LogFile._header to
        create a header.
        """
        for k in self.MsgListLog:
            for n in k:
                contents = contents + str(n) + ';'
            # at the end of each line, new line symbol
            contents = contents + '\n'
        return contents

    def csv2df(self, fname):
        """Read a csv log file analysis and convert to a pandas.DataFrame
        """
        colnames, min_itemsize, dtypes = self.headers4df()
        df = pd.read_csv(fname, header=0, names=colnames, sep=';', )
        for col, dtype in dtypes.items():
            df[col] = df[col].astype(dtype)
            # replace nan with empty for str columns
            if dtype == str:
                df[col] = df[col].str.replace('nan', '')
        return df

    def _header(self):
        """Header for log analysis csv file
        """

        # write the results in a file, start with a header
        contents = 'file name;' + 'nr;msg;'*(self.init_cols)
        contents += 'first_tstep;last_tstep;nr;msg;'*(self.sim_cols)
        contents += 'lnr;msg;'
        # and add headers for elapsed time, nr of iterations, and sec/iteration
        contents += 'Elapsted time;last time step;Simulation time;'
        contents += 'real sim time;Sim output time;'
        contents += 'total iterations;dt;nr time steps;'
        contents += 'seconds/iteration;average iterations/time step;'
        contents += 'maximum iterations/time step;\n'

        return contents

    def headers4df(self):
        """Create header and a minimum itemsize for string columns when
        converting a Log check analysis to a pandas.DataFrame

        Returns
        -------

        header : list
            List of column names as generated by WindIO.LogFile._header

        min_itemsize : dict
            Dictionary with column names as keys, and the minimum string lenght
            as values.

        dtypes : dict
            Dictionary with column names as keys, and data types as values
        """
        chain_iter = chain.from_iterable

        colnames = ['file_name']
        colnames.extend(list(chain_iter(('nr_%i' % i, 'msg_%i' % i)
                      for i in range(31))) )

        gr = ('first_tstep_%i', 'last_step_%i', 'nr_%i', 'msg_%i')
        colnames.extend(list(chain_iter( (k % i for k in gr)
                           for i in range(100,105,1))) )
        colnames.extend(['nr_extra', 'msg_extra'])
        colnames.extend(['elapsted_time',
                       'last_time_step',
                       'simulation_time',
                       'real_sim_time',
                       'sim_output_time',
                       'total_iterations',
                       'dt',
                       'nr_time_steps',
                       'seconds_p_iteration',
                       'mean_iters_p_time_step',
                       'max_iters_p_time_step',
                       'sim_id'])
        dtypes = {}

        # str and float datatypes for
        msg_cols = ['msg_%i' % i for i in range(30)]
        msg_cols.extend(['msg_%i' % i for i in range(100,105,1)])
        msg_cols.append('msg_extra')
        dtypes.update({k:str for k in msg_cols})
        # make the message/str columns long enough
        min_itemsize = {'msg_%i' % i : 100 for i in range(30)}

        # column names holding the number of occurances of messages
        nr_cols = ['nr_%i' % i for i in range(30)]
        nr_cols.extend(['nr_%i' % i for i in range(100,105,1)])
        # other float values
        nr_cols.extend(['elapsted_time', 'total_iterations'])
        # NaN only exists in float arrays, not integers (NumPy limitation)
        # so use float instead of int
        dtypes.update({k:np.float64 for k in nr_cols})

        return colnames, min_itemsize, dtypes


class LoadResults(ReadHawc2):
    """Read a HAWC2 result data file

    Usage:
    obj = LoadResults(file_path, file_name)

    This class is called like a function:
    HawcResultData() will read the specified file upon object initialization.

    Available output:
    obj.sig[timeStep,channel]   : complete result file in a numpy array
    obj.ch_details[channel,(0=ID; 1=units; 2=description)] : np.array
    obj.error_msg: is 'none' if everything went OK, otherwise it holds the
    error

    The ch_dict key/values pairs are structured differently for different
        type of channels. Currently supported channels are:

        For forcevec, momentvec, state commands:
            key:
                coord-bodyname-pos-sensortype-component
                global-tower-node-002-forcevec-z
                local-blade1-node-005-momentvec-z
                hub1-blade1-elem-011-zrel-1.00-state pos-z
            value:
                ch_dict[tag]['coord']
                ch_dict[tag]['bodyname']
                ch_dict[tag]['pos'] = pos
                ch_dict[tag]['sensortype']
                ch_dict[tag]['component']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the DLL's this is:
            key:
                DLL-dll_name-io-io_nr
                DLL-yaw_control-outvec-3
                DLL-yaw_control-inpvec-1
            value:
                ch_dict[tag]['dll_name']
                ch_dict[tag]['io']
                ch_dict[tag]['io_nr']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the bearings this is:
            key:
                bearing-bearing_name-output_type-units
                bearing-shaft_nacelle-angle_speed-rpm
            value:
                ch_dict[tag]['bearing_name']
                ch_dict[tag]['output_type']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']

    """
    # ch_df columns, these are created by LoadResults._unified_channel_names
    cols = set(['bearing_name', 'sensortag', 'bodyname', 'chi', 'component',
                'pos', 'coord', 'sensortype', 'radius', 'blade_nr', 'units',
                'output_type', 'io_nr', 'io', 'dll', 'azimuth', 'flap_nr',
                'direction'])

    # start with reading the .sel file, containing the info regarding
    # how to read the binary file and the channel information
    def __init__(self, file_path, file_name, debug=False, usecols=None,
                 readdata=True):

        self.debug = debug

        # timer in debug mode
        if self.debug:
            start = time()

        self.file_path = file_path
        # remove .log, .dat, .sel extensions who might be accedental left
        if file_name[-4:] in ['.htc', '.sel', '.dat', '.log']:
            file_name = file_name[:-4]
        # FIXME: since HAWC2 will always have lower case output files, convert
        # any wrongly used upper case letters to lower case here
        self.file_name = file_name
        FileName = os.path.join(self.file_path, self.file_name)

        ReadOnly = 0 if readdata else 1
        super(LoadResults, self).__init__(FileName, ReadOnly=ReadOnly)
        self.FileType = self.FileFormat[6:]
        self.N = int(self.NrSc)
        self.Nch = int(self.NrCh)
        self.ch_details = np.ndarray(shape=(self.Nch, 3), dtype='<U100')
        for ic in range(self.Nch):
            self.ch_details[ic, 0] = self.ChInfo[0][ic]
            self.ch_details[ic, 1] = self.ChInfo[1][ic]
            self.ch_details[ic, 2] = self.ChInfo[2][ic]

        ChVec = [] if usecols is None else usecols

        self._unified_channel_names()
        if readdata:
            self.sig = super(LoadResults, self).__call__(ChVec=ChVec)

        if self.debug:
            stop = time() - start
            print('time to load HAWC2 file:', stop, 's')


    def reformat_sig_details(self):
        """Change HAWC2 output description of the channels short descriptive
        strings, usable in plots

        obj.ch_details[channel,(0=ID; 1=units; 2=description)] : np.array
        """

        # CONFIGURATION: mappings between HAWC2 and short good output:
        change_list = []
        change_list.append( ['original', 'new improved'] )

#        change_list.append( ['Mx coo: hub1','blade1 root bending: flap'] )
#        change_list.append( ['My coo: hub1','blade1 root bending: edge'] )
#        change_list.append( ['Mz coo: hub1','blade1 root bending: torsion'] )
#
#        change_list.append( ['Mx coo: hub2','blade2 root bending: flap'] )
#        change_list.append( ['My coo: hub2','blade2 root bending: edge'] )
#        change_list.append( ['Mz coo: hub2','blade2 root bending: torsion'] )
#
#        change_list.append( ['Mx coo: hub3','blade3 root bending: flap'] )
#        change_list.append( ['My coo: hub3','blade3 root bending: edge'] )
#        change_list.append( ['Mz coo: hub3','blade3 root bending: torsion'] )

        change_list.append(['Mx coo: blade1', 'blade1 flap'])
        change_list.append(['My coo: blade1', 'blade1 edge'])
        change_list.append(['Mz coo: blade1', 'blade1 torsion'])

        change_list.append(['Mx coo: blade2', 'blade2 flap'])
        change_list.append(['My coo: blade2', 'blade2 edge'])
        change_list.append(['Mz coo: blade2', 'blade2 torsion'])

        change_list.append(['Mx coo: blade3', 'blade3 flap'])
        change_list.append(['My coo: blade3', 'blade3 edeg'])
        change_list.append(['Mz coo: blade3', 'blade3 torsion'])

        change_list.append(['Mx coo: hub1', 'blade1 out-of-plane'])
        change_list.append(['My coo: hub1', 'blade1 in-plane'])
        change_list.append(['Mz coo: hub1', 'blade1 torsion'])

        change_list.append(['Mx coo: hub2', 'blade2 out-of-plane'])
        change_list.append(['My coo: hub2', 'blade2 in-plane'])
        change_list.append(['Mz coo: hub2', 'blade2 torsion'])

        change_list.append(['Mx coo: hub3', 'blade3 out-of-plane'])
        change_list.append(['My coo: hub3', 'blade3 in-plane'])
        change_list.append(['Mz coo: hub3', 'blade3 torsion'])
        # this one will create a false positive for tower node nr1
        change_list.append(['Mx coo: tower', 'tower top momemt FA'])
        change_list.append(['My coo: tower', 'tower top momemt SS'])
        change_list.append(['Mz coo: tower', 'yaw-moment'])

        change_list.append(['Mx coo: chasis', 'chasis momemt FA'])
        change_list.append(['My coo: chasis', 'yaw-moment chasis'])
        change_list.append(['Mz coo: chasis', 'chasis moment SS'])

        change_list.append(['DLL inp  2:  2', 'tower clearance'])

        self.ch_details_new = np.ndarray(shape=(self.Nch, 3), dtype='<U100')

        # approach: look for a specific description and change it.
        # This approach is slow, but will not fail if the channel numbers change
        # over different simulations
        for ch in range(self.Nch):
            # the change_list will always be slower, so this loop will be
            # inside the bigger loop of all channels
            self.ch_details_new[ch, :] = self.ch_details[ch, :]
            for k in range(len(change_list)):
                if change_list[k][0] == self.ch_details[ch, 0]:
                    self.ch_details_new[ch, 0] = change_list[k][1]
                    # channel description should be unique, so delete current
                    # entry and stop looking in the change list
                    del change_list[k]
                    break

    # TODO: THIS IS STILL A WIP
    def _make_channel_names(self):
        """Give every channel a unique channel name which is (nearly) identical
        to the channel names as defined in the htc output section. Instead
        of spaces, use colon (;) to seperate the different commands.

        THIS IS STILL A WIP
        """

        index = {}

        names = {'htc_name':[], 'chi':[], 'label':[], 'unit':[], 'index':[],
                 'name':[], 'description':[]}
        constraint_fmts = {'bea1':'constraint;bearing1',
                           'bea2':'constraint;bearing2',
                           'bea3':'constraint;bearing3',
                           'bea4':'constraint;bearing4'}
        # mbdy momentvec tower  1 1 global
        force_fmts = {'F':'mbdy;forcevec;{body};{nodenr:03i};{coord};{comp}',
                      'M':'mbdy;momentvec;{body};{nodenr:03i};{coord};{comp}'}
        state_fmt = 'mbdy;{state};{typ};{body};{elnr:03i};{zrel:01.02f};{coord}'

        wind_coord_map = {'Vx':'1', 'Vy':'2', 'Vz':'3'}
        wind_fmt = 'wind;{typ};{coord};{x};{y};{z};{comp}'

        for ch in range(self.Nch):
            name = self.ch_details[ch, 0]
            name_items = misc.remove_items(name.split(' '), '')

            description = self.ch_details[ch, 2]
            descr_items = misc.remove_items(description.split(' '), '')

            unit = self.ch_details[ch, 1]

            # default names
            htc_name = ' '.join(name_items+descr_items)
            label = ''
            coord = ''
            typ = ''
            elnr = ''
            nodenr = ''
            zrel = ''
            state = ''

            # CONSTRAINTS: BEARINGS
            if name_items[0] in constraint_fmts:
                htc_name = constraint_fmts[name_items[0]] + ';'
                htc_name += (descr_items[0] + ';')
                htc_name += unit

            # MBDY FORCES/MOMENTS
            elif name_items[0][0] in force_fmts:
                comp = name_items[0]
                if comp[0] == 'F':
                    i0 = 1
                else:
                    i0 = 0
                label = description.split('coo: ')[1].split('  ')[1]
                coord = descr_items[i0+5]
                body = descr_items[i0+1][5:]#.replace('Mbdy:', '')
                nodenr = int(descr_items[i0+3])
                htc_name = force_fmts[comp[0]].format(body=body, coord=coord,
                                                      nodenr=nodenr, comp=comp)

            # STATE: POS, VEL, ACC, STATE_ROT
            elif descr_items[0][:5] == 'State':
                if name_items[0] == 'State':
                    i0 = 1
                    state = 'state'
                else:
                    i0 = 0
                    state = 'state_rot'
                typ = name_items[i0+0]
                comp = name_items[i0+1]
                coord = name_items[i0+3]

                body = descr_items[3][5:]#.replace('Mbdy:', '')
                elnr = int(descr_items[5])
                zrel = float(descr_items[6][6:])#.replace('Z-rel:', ''))
                if len(descr_items) > 8:
                    label = ' '.join(descr_items[9:])
                htc_name = state_fmt.format(typ=typ, body=body, elnr=elnr,
                                            zrel=zrel, coord=coord,
                                            state=state)

            # WINDSPEED
            elif description[:9] == 'Free wind':
                if descr_items[4] == 'gl.':
                    coord = '1' # global
                else:
                    coord = '2' # non-rotating rotor coordinates

                try:
                    comp = wind_coord_map[descr_items[3][:-1]]
                    typ = 'free_wind'
                except KeyError:
                    comp = descr_items[3]
                    typ = 'free_wind_hor'

                tmp = description.split('pos')[1]
                x, y, z = tmp.split(',')
                # z might hold a label....
                z_items  = z.split('  ')
                if len(z_items) > 1:
                    label = '  '.join(z_items[1:])
                    z = z_items[0]
                x, y, z = x.strip(), y.strip(), z.strip()

                htc_name = wind_fmt.format(typ=typ, coord=coord, x=x, y=y, z=z,
                                           comp=comp)


            names['htc_name'].append(htc_name)
            names['chi'].append(ch)
            # this is the Channel column from the sel file, so the unique index
            # which is dependent on the order of the channels
            names['index'].append(ch+1)
            names['unit'].append(unit)
            names['name'].append(name)
            names['description'].append(description)
            names['label'].append(label)
            names['state'].append(state)
            names['type'].append(typ)
            names['comp'].append(comp)
            names['coord'].append(coord)
            names['elnr'].append(coord)
            names['nodenr'].append(coord)
            names['zrel'].append(coord)
            index[name] = ch

        return names, index


    def _unified_channel_names(self):
        """
        Make certain channels independent from their index.

        The unified channel dictionary ch_dict holds consequently named
        channels as the key, and the all information is stored in the value
        as another dictionary.

        The ch_dict key/values pairs are structured differently for different
        type of channels. Currently supported channels are:

        For forcevec, momentvec, state commands:
            node numbers start with 0 at the root
            element numbers start with 1 at the root
            key:
                coord-bodyname-pos-sensortype-component
                global-tower-node-002-forcevec-z
                local-blade1-node-005-momentvec-z
                hub1-blade1-elem-011-zrel-1.00-state pos-z
            value:
                ch_dict[tag]['coord']
                ch_dict[tag]['bodyname']
                ch_dict[tag]['pos']
                ch_dict[tag]['sensortype']
                ch_dict[tag]['component']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the DLL's this is:
            key:
                DLL-dll_name-io-io_nr
                DLL-yaw_control-outvec-3
                DLL-yaw_control-inpvec-1
            value:
                ch_dict[tag]['dll_name']
                ch_dict[tag]['io']
                ch_dict[tag]['io_nr']
                ch_dict[tag]['chi']
                ch_dict[tag]['sensortag']
                ch_dict[tag]['units']

        For the bearings this is:
            key:
                bearing-bearing_name-output_type-units
                bearing-shaft_nacelle-angle_speed-rpm
            value:
                ch_dict[tag]['bearing_name']
                ch_dict[tag]['output_type']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']

        For many of the aero sensors:
            'Cl', 'Cd', 'Alfa', 'Vrel'
            key:
                sensortype-blade_nr-pos
                Cl-1-0.01
            value:
                ch_dict[tag]['sensortype']
                ch_dict[tag]['blade_nr']
                ch_dict[tag]['pos']
                ch_dict[tag]['chi']
                ch_dict[tag]['units']
        """
        # save them in a dictionary, use the new coherent naming structure
        # as the key, and as value again a dict that hols all the different
        # classifications: (chi, channel nr), (coord, coord), ...
        self.ch_dict = dict()

        # some channel ID's are unique, use them
        ch_unique = set(['Omega', 'Ae rot. torque', 'Ae rot. power',
                         'Ae rot. thrust', 'Time', 'Azi  1'])
        ch_aero = set(['Cl', 'Cd', 'Alfa', 'Vrel', 'Tors_e', 'Alfa'])
        ch_aerogrid = set(['a_grid', 'am_grid'])

        # also safe as df
#        cols = set(['bearing_name', 'sensortag', 'bodyname', 'chi',
#                    'component', 'pos', 'coord', 'sensortype', 'radius',
#                    'blade_nr', 'units', 'output_type', 'io_nr', 'io', 'dll',
#                    'azimuth', 'flap_nr'])
        df_dict = {col: [] for col in self.cols}
        df_dict['unique_ch_name'] = []

        # scan through all channels and see which can be converted
        # to sensible unified name
        for ch in range(self.Nch):
            items = self.ch_details[ch, 2].split(' ')
            # remove empty values in the list
            items = misc.remove_items(items, '')

            dll = False

            # be carefull, identify only on the starting characters, because
            # the signal tag can hold random text that in some cases might
            # trigger a false positive

            # -----------------------------------------------------------------
            # check for all the unique channel descriptions
            if self.ch_details[ch,0].strip() in ch_unique:
                tag = self.ch_details[ch, 0].strip()
                channelinfo = {}
                channelinfo['units'] = self.ch_details[ch, 1]
                channelinfo['sensortag'] = self.ch_details[ch, 2]
                channelinfo['chi'] = ch

            # -----------------------------------------------------------------
            # or in the long description:
            #    0          1        2      3  4    5     6 and up
            # MomentMz Mbdy:blade nodenr:   5 coo: blade  TAG TEXT
            elif self.ch_details[ch, 2].startswith('MomentM'):
                coord = items[5]
                bodyname = items[1].replace('Mbdy:', '')
                # set nodenr to sortable way, include leading zeros
                # node numbers start with 0 at the root
                nodenr = '%03i' % int(items[3])
                # skip the attached the component
                # sensortype = items[0][:-2]
                # or give the sensor type the same name as in HAWC2
                sensortype = 'momentvec'
                component = items[0][-1:len(items[0])]
                # the tag only exists if defined
                if len(items) > 6:
                    sensortag = ' '.join(items[6:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'node-%s' % nodenr
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            #   0    1      2        3       4  5     6     7 and up
            # Force  Fx Mbdy:blade nodenr:   2 coo: blade  TAG TEXT
            elif self.ch_details[ch, 2].startswith('Force'):
                coord = items[6]
                bodyname = items[2].replace('Mbdy:', '')
                nodenr = '%03i' % int(items[4])
                # skipe the attached the component
                # sensortype = items[0]
                # or give the sensor type the same name as in HAWC2
                sensortype = 'forcevec'
                component = items[1][1]
                if len(items) > 7:
                    sensortag = ' '.join(items[7:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'node-%s' % nodenr
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            #   0    1  2      3       4      5   6         7    8
            # State pos x  Mbdy:blade E-nr:   1 Z-rel:0.00 coo: blade
            #   0           1     2    3        4    5   6         7     8     9+
            # State_rot proj_ang tx Mbdy:bname E-nr: 1 Z-rel:0.00 coo: cname  label
            # State_rot omegadot tz Mbdy:bname E-nr: 1 Z-rel:1.00 coo: cname  label
            elif self.ch_details[ch,2].startswith('State'):
#                 or self.ch_details[ch,0].startswith('euler') \
#                 or self.ch_details[ch,0].startswith('ax') \
#                 or self.ch_details[ch,0].startswith('omega') \
#                 or self.ch_details[ch,0].startswith('proj'):
                coord = items[8]
                bodyname = items[3].replace('Mbdy:', '')
                # element numbers start with 1 at the root
                elementnr = '%03i' % int(items[5])
                zrel = '%04.2f' % float(items[6].replace('Z-rel:', ''))
                # skip the attached the component
                #sensortype = ''.join(items[0:2])
                # or give the sensor type the same name as in HAWC2
                tmp = self.ch_details[ch, 0].split(' ')
                sensortype = tmp[0]
                if sensortype.startswith('State'):
                    sensortype += ' ' + tmp[1]
                component = items[2]
                if len(items) > 8:
                    sensortag = ' '.join(items[9:])
                else:
                    sensortag = ''

                # and tag it
                pos = 'elem-%s-zrel-%s' % (elementnr, zrel)
                tagitems = (coord, bodyname, pos, sensortype, component)
                tag = '%s-%s-%s-%s-%s' % tagitems
                # save all info in the dict
                channelinfo = {}
                channelinfo['coord'] = coord
                channelinfo['bodyname'] = bodyname
                channelinfo['pos'] = pos
                channelinfo['sensortype'] = sensortype
                channelinfo['component'] = component
                channelinfo['chi'] = ch
                channelinfo['sensortag'] = sensortag
                channelinfo['units'] = self.ch_details[ch, 1]

            # -----------------------------------------------------------------
            # DLL CONTROL I/O
            # there are two scenario's on how the channel description is formed
            # the channel id is always the same though
            # id for all three cases:
            #          DLL out  1:  3
            #          DLL inp  2:  3
            # description case 1 ("dll type2_dll b2h2 inpvec 30" in htc output)
            #               0         1    2   3     4+
            #          yaw_control outvec  3  yaw_c input reference angle
            # description case 2 ("dll inpvec 2 1" in htc output):
            #           0  1 2     3  4  5  6+