Newer
Older
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 5 14:01:25 2014
@author: dave
"""
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from builtins import str
from future.utils import viewitems
from future import standard_library
standard_library.install_aliases()
import os
import unittest
import pandas as pd
from wetb.prepost import misc
def casedict2xlsx():
"""
Convert a full Cases.cases dict to Excel spreadsheets
"""
def configure_dirs(verbose=False):
"""
Automatically configure required directories to launch simulations
"""
p_run_root = os.sep.join(P_RUN.split(os.sep)[:-2])
# MODEL SOURCES, exchanche file sources
P_SOURCE = P_RUN
# Project name, sim_id: derive from folder name
PROJECT = P_RUN.split(os.sep)[-2]
sim_id = P_RUN.split(os.sep)[-1]
master = find_master_file(P_SOURCE)
if master is None:
raise ValueError('Could not find master file in htc/_master')
MASTERFILE = master
P_MASTERFILE = os.path.join(P_SOURCE, 'htc%s_master%s' % (os.sep, os.sep))
POST_DIR = os.path.join(p_run_root, PROJECT, sim_id, 'prepost-data%s' % os.sep)
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
if verbose:
print('='*79)
print('POST_DIR: %s' % POST_DIR)
print(' P_RUN: %s' % P_RUN)
print('P_SOURCE: %s' % P_SOURCE)
print(' PROJECT: %s' % PROJECT)
print(' sim_id: %s' % sim_id)
print(' master: %s' % MASTERFILE)
print('='*79)
return P_RUN, P_SOURCE, PROJECT, sim_id, P_MASTERFILE, MASTERFILE, POST_DIR
def find_master_file(proot, htc_dir='htc', master_dir='_master',
master_contains='_master_'):
"""
Find the master file name. It is assumed that the master file is in the
folder _master, under htc, and contains _master_ in the file name.
"""
for root, dirs, files in os.walk(os.path.join(proot, htc_dir, master_dir)):
for fname in files:
if fname.find(master_contains) > -1:
return fname
return None
def variable_tag_func(master, case_id_short=False):
"""
When using the Excel definitions, and the whole default setup, the
variable_tag_func is not required to do anything extra.
"""
# -------------------------------------------------------------------------
# mt = master
# V = mt['windspeed']
# mt['duration'] = mt['time_stop'] - mt['t0']
# t = mt['duration']
# if V > abs(1e-15):
# b = 5.6
# mt['TI'] = mt['TI_ref'] * ((0.75*V) + b) / V # NTM
# # ETM
# c = 2.0
# V_ave = 0.2 * 50.0
# sigma = mt['TI_ref'] / V
# mt['TI'] = sigma * c * (0.072 * (V_ave / c + 3.0) * (V / c - 4.0) + 10.0)
# else:
# mt['TI'] = 0
#
# mt['turb_dx'] = V*t/mt['turb_grid_x']
#
# mt['turb_dy'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# mt['turb_dz'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# # check: dx spacing should be 0.1*mean_windspeed and 0.2*mean_windspeed
# # between 0.1 and 0.2 seconds between points
# if not (V*0.1 < mt['turb_dx'] < V*0.2):
# logging.warn('turbulence spacing dx out of bounds')
# print('%5.3f %5.3f %5.3f' % (V*0.1, mt['turb_dx'], V*0.2))
#
# #mt['turb_base_name'] = 'turb_s' + str(mt['turb_seed']) + '_' + str(V)
# mt['turb_base_name'] = 'turb_s%i_%1.2f' % (mt['turb_seed'], V)
# -------------------------------------------------------------------------
return master
def vartags_dlcs(master):
mt = master.tags
dlc_case = mt['[Case folder]']
mt['[data_dir]'] = 'data/'
mt['[res_dir]'] = 'res/%s/' % dlc_case
mt['[log_dir]'] = 'logfiles/%s/' % dlc_case
mt['[htc_dir]'] = 'htc/%s/' % dlc_case
mt['[case_id]'] = mt['[Case id.]']
mt['[time_stop]'] = mt['[time stop]']
mt['[turb_base_name]'] = mt['[Turb base name]']
mt['[DLC]'] = mt['[Case id.]'].split('_')[0][3:]
mt['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
mt['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
mt['[iter_dir]'] = 'iter/%s/' % dlc_case
if mt['[eigen_analysis]']:
rpl = (dlc_case, mt['[Case id.]'])
mt['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
mt['[duration]'] = str(float(mt['[time_stop]']) - float(mt['[t0]']))
# replace nan with empty
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
if jj == 'nan':
mt[ii] = ''
return master
def tags_dlcs(master):
"""
Initiate tags that are defined in the DLC spreadsheets
"""
master.tags['[t0]'] = 0
master.tags['[time stop]'] = 0
master.tags['[Case folder]'] = 'test'
master.tags['[Case id.]'] = 'test'
master.tags['[Windspeed]'] = 8
master.tags['[wdir]'] = 0 # used for the user defined wind
master.tags['[wdir_rot]'] = 0 # used for the windfield rotations
master.tags['[tu_seed]'] = 0
master.tags['[tu_model]'] = 0
master.tags['[TI]'] = 0
master.tags['[Turb base name]'] = 'none'
master.tags['[turb_dx]'] = 1.0
master.tags['[shear_exp]'] = 0.2
master.tags['[wsp factor]'] = 1.0
master.tags['[gust]'] = False
master.tags['[gust_type]'] = ''
master.tags['[G_A]'] = ''
master.tags['[G_phi0]'] = ''
master.tags['[G_t0]'] = ''
master.tags['[G_T]'] = ''
master.tags['[Rotor azimuth]'] = 0
master.tags['[Free shaft rot]'] = ''
master.tags['[init_wr]'] = 0.5
master.tags['[Pitch 1 DLC22b]'] = 0
master.tags['[Rotor locked]'] = False
master.tags['[Time stuck DLC22b]'] = -1
master.tags['[Cut-in time]'] = -1
master.tags['[Cut-out time]'] = -1
master.tags['[Stop type]'] = -1
master.tags['[Pitvel 1]'] = 4
master.tags['[Pitvel 2]'] = 6
master.tags['[Grid loss time]'] = 1000
master.tags['[out_format]'] = 'hawc_binary'
master.tags['[Time pitch runaway]'] = 1000
master.tags['[Induction]'] = 1
master.tags['[Dyn stall]'] = 1
# required tags for the MannTurb64 standalone turbulence box generator
master.tags['[MannAlfaEpsilon]'] = 1.0
master.tags['[MannL]'] = 29.4
master.tags['[MannGamma]'] = 3.0
master.tags['[turb_nr_u]'] = 8192
master.tags['[turb_nr_v]'] = 32
master.tags['[turb_nr_w]'] = 32
master.tags['[turb_dx]'] = 1
master.tags['[turb_dy]'] = 6.5
master.tags['[turb_dz]'] = 6.5
master.tags['[high_freq_comp]'] = 1
return master
def tags_defaults(master):
# other required tags and their defaults
master.tags['[dt_sim]'] = 0.02
master.tags['[hawc2_exe]'] = 'hawc2-latest'
# folder names for the saved results, htc, data, zip files
# Following dirs are relative to the model_dir_server and they specify
# the location of where the results, logfiles, animation files that where
# run on the server should be copied to after the simulation has finished.
# on the node, it will try to copy the turbulence files from these dirs
master.tags['[animation_dir]'] = 'animation/'
master.tags['[control_dir]'] = 'control/'
master.tags['[data_dir]'] = 'data/'
master.tags['[eigen_analysis]'] = False
master.tags['[eigenfreq_dir]'] = False
master.tags['[htc_dir]'] = 'htc/'
master.tags['[log_dir]'] = 'logfiles/'
master.tags['[meander_dir]'] = False
master.tags['[opt_dir]'] = False
master.tags['[pbs_out_dir]'] = 'pbs_out/'
master.tags['[res_dir]'] = 'res/'
master.tags['[iter_dir]'] = 'iter/'
master.tags['[turb_dir]'] = 'turb/'
master.tags['[turb_db_dir]'] = '../turb/'
master.tags['[wake_dir]'] = False
master.tags['[hydro_dir]'] = False
master.tags['[mooring_dir]'] = False
master.tags['[externalforce]'] = False
# required tags for the MannTurb64 standalone turbulence box generator
master.tags['[MannAlfaEpsilon]'] = 1.0
master.tags['[MannL]'] = 29.4
master.tags['[MannGamma]'] = 3.0
master.tags['[turb_nr_u]'] = 8192
master.tags['[turb_nr_v]'] = 32
master.tags['[turb_nr_w]'] = 32
master.tags['[turb_dx]'] = 1
master.tags['[turb_dy]'] = 6.5
master.tags['[turb_dz]'] = 6.5
master.tags['[high_freq_comp]'] = 1
# zip_root_files only is used when copy to run_dir and zip creation, define
# in the HtcMaster object
master.tags['[zip_root_files]'] = []
# only active on PBS level, so files have to be present in the run_dir
master.tags['[copyback_files]'] = [] # copyback_resultfile
master.tags['[copyback_frename]'] = [] # copyback_resultrename
master.tags['[copyto_files]'] = [] # copyto_inputfile
master.tags['[copyto_generic]'] = [] # copyto_input_required_defaultname
master.tags['[eigen_analysis]'] = False
# =========================================================================
# basic required tags by HtcMaster and PBS in order to function properly
# =========================================================================
# the express queue ('#PBS -q xpresq') has a maximum walltime of 1h
master.tags['[pbs_queue_command]'] = '#PBS -q workq'
# walltime should have following format: hh:mm:ss
master.tags['[walltime]'] = '04:00:00'
master.tags['[auto_walltime]'] = False
return master
def excel_stabcon(proot, fext='xlsx', pignore=None, pinclude=None, sheet=0,
silent=False):
"""
Read all MS Excel files that hold load case definitions according to
the team STABCON definitions. Save each case in a list according to the
opt_tags principles as used in Simulations.launch(). This method assumes
that a standard HAWC2 folder layout is used with the following folder
names: res, logfiles, htc, pbs_out, pbs_in, iter. Further some tags
are added to be compatible with the tag convention in the Simulations
module.
The opt_tags case list is sorted according to the Excel file names, and
follows the same ordering as in each of the different Excel files.
Parameters
----------
proot : string
Path that will be searched recursively for Excel files containing
load case definitions.
fext : string, default='xlsx'
File extension of the Excel files that should be loaded
pignore : string, default=None
Specify which string can not occur in the full path of the DLC target.
pinclude : string, default=None
Specify which string has to occur in the full path of the DLC target.
sheet : string or int, default=0
Name or index of the Excel sheet to be considered. By default, the
first sheet (index=0) is taken.
Returns
-------
opt_tags : list of dicts
A list of case dictionaries, where each case dictionary holds all
the tag/value key pairs for a single given case.
if not silent:
print('looking for DLC spreadsheet definitions at:')
print(proot)
dict_dfs = misc.read_excel_files(proot, fext=fext, pignore=pignore,
sheet=sheet, pinclude=pinclude,
silent=silent)
print('found %i Excel file(s), ' % len(dict_dfs), end='')
for df in dict_dfs:
if not silent:
print('in which a total of %s cases are defined.' % k)
for (dlc, df) in sorted(viewitems(dict_dfs)):
# replace ';' with False, and Nan(='') with True
# this is more easy when testing for the presence of stuff compared
# to checking if a value is either True/False or ''/';'
# this doesn't work, it will result in 1 for True and 0 for False
# because the nan values have np.float dtype
# df.fillna(' ', inplace=True)
# df.replace(';', False, inplace=True)
# instead, convert everything to strings, this will maintain some nans
# as empty strings, but not all of them!
df2 = df.astype(str)
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.iteritems():
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
if isinstance(tags_dict[str(key)], str):
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
tags_dict['[Case folder]'] = tags_dict['[Case folder]'].lower()
tags_dict['[Case id.]'] = tags_dict['[Case id.]'].lower()
dlc_case = tags_dict['[Case folder]']
tags_dict['[data_dir]'] = 'data/'
tags_dict['[res_dir]'] = 'res/%s/' % dlc_case
tags_dict['[log_dir]'] = 'logfiles/%s/' % dlc_case
tags_dict['[htc_dir]'] = 'htc/%s/' % dlc_case
if '[Case id.]' in tags_dict.keys():
tags_dict['[case_id]'] = tags_dict['[Case id.]']
if '[time stop]' in tags_dict.keys():
tags_dict['[time_stop]'] = tags_dict['[time stop]']
try:
tags_dict['[turb_base_name]'] = tags_dict['[Turb base name]']
except KeyError:
tags_dict['[turb_base_name]'] = None
tags_dict['[Turb base name]'] = None
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
tags_dict['[DLC]'] = tags_dict['[Case id.]'].split('_')[0][3:]
tags_dict['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
tags_dict['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
tags_dict['[iter_dir]'] = 'iter/%s/' % dlc_case
# the default spreadsheets do not define the tags related to the
# eigen analsyis yet
if '[eigen_analysis]' in tags_dict and tags_dict['[eigen_analysis]']:
rpl = (dlc_case, tags_dict['[Case id.]'])
if '[eigenfreq_dir]' in tags_dict:
tags_dict['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
t_stop = float(tags_dict['[time_stop]'])
t0 = float(tags_dict['[t0]'])
tags_dict['[duration]'] = str(t_stop - t0)
opt_tags.append(tags_dict.copy())
return opt_tags
def read_tags_spreadsheet(fname):
"""Read a spreadsheet with HAWC2 tags, make sure no 0/1/nan ends up
replacing the ";" or "" (empty). Do not add any other tags.
Returns
-------
opt_tags : [{}, {}] list of dictionaries
"""
df = pd.read_excel(fname)
df2 = df.astype(str)
opt_tags = []
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.items():
if isinstance(value, str):
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
opt_tags.append(tags_dict.copy())
return opt_tags
class Tests(unittest.TestCase):
"""
"""
def setUp(self):
self.fpath = 'data/DLCs'
def test_read_tag_exchange_file(self):
df_list = misc.read_excel_files(self.fpath, fext='xlsx', pignore=None,
sheet=0, pinclude=None)
df = df_list[list(df_list.keys())[0]]