Newer
Older
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 5 14:01:25 2014
@author: dave
"""
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from builtins import str
from future.utils import viewitems
from future import standard_library
standard_library.install_aliases()
import os
import unittest
import pandas as pd
from wetb.prepost import misc
def casedict2xlsx():
"""
Convert a full Cases.cases dict to Excel spreadsheets
"""
def configure_dirs(verbose=False):
"""
Automatically configure required directories to launch simulations
"""
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
p_run_root = os.sep.join(P_RUN.split(os.sep)[:-2])
# MODEL SOURCES, exchanche file sources
P_SOURCE = P_RUN
# Project name, sim_id: derive from folder name
PROJECT = P_RUN.split(os.sep)[-2]
sim_id = P_RUN.split(os.sep)[-1]
master = find_master_file(P_SOURCE)
if master is None:
raise ValueError('Could not find master file in htc/_master')
MASTERFILE = master
P_MASTERFILE = os.path.join(P_SOURCE, 'htc%s_master%s' % (os.sep, os.sep))
POST_DIR = os.path.join(p_run_root, PROJECT, 'python-prepost-data%s' % os.sep)
if verbose:
print('='*79)
print('POST_DIR: %s' % POST_DIR)
print(' P_RUN: %s' % P_RUN)
print('P_SOURCE: %s' % P_SOURCE)
print(' PROJECT: %s' % PROJECT)
print(' sim_id: %s' % sim_id)
print(' master: %s' % MASTERFILE)
print('='*79)
return P_RUN, P_SOURCE, PROJECT, sim_id, P_MASTERFILE, MASTERFILE, POST_DIR
def find_master_file(proot, htc_dir='htc', master_dir='_master',
master_contains='_master_'):
"""
Find the master file name. It is assumed that the master file is in the
folder _master, under htc, and contains _master_ in the file name.
"""
for root, dirs, files in os.walk(os.path.join(proot, htc_dir, master_dir)):
for fname in files:
if fname.find(master_contains) > -1:
return fname
return None
def variable_tag_func(master, case_id_short=False):
"""
When using the Excel definitions, and the whole default setup, the
variable_tag_func is not required to do anything extra.
"""
# -------------------------------------------------------------------------
# mt = master
# V = mt['windspeed']
# mt['duration'] = mt['time_stop'] - mt['t0']
# t = mt['duration']
# if V > abs(1e-15):
# b = 5.6
# mt['TI'] = mt['TI_ref'] * ((0.75*V) + b) / V # NTM
# # ETM
# c = 2.0
# V_ave = 0.2 * 50.0
# sigma = mt['TI_ref'] / V
# mt['TI'] = sigma * c * (0.072 * (V_ave / c + 3.0) * (V / c - 4.0) + 10.0)
# else:
# mt['TI'] = 0
#
# mt['turb_dx'] = V*t/mt['turb_grid_x']
#
# mt['turb_dy'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# mt['turb_dz'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# # check: dx spacing should be 0.1*mean_windspeed and 0.2*mean_windspeed
# # between 0.1 and 0.2 seconds between points
# if not (V*0.1 < mt['turb_dx'] < V*0.2):
# logging.warn('turbulence spacing dx out of bounds')
# print('%5.3f %5.3f %5.3f' % (V*0.1, mt['turb_dx'], V*0.2))
#
# #mt['turb_base_name'] = 'turb_s' + str(mt['turb_seed']) + '_' + str(V)
# mt['turb_base_name'] = 'turb_s%i_%1.2f' % (mt['turb_seed'], V)
# -------------------------------------------------------------------------
return master
def vartags_dlcs(master):
mt = master.tags
dlc_case = mt['[Case folder]']
mt['[data_dir]'] = 'data/'
mt['[res_dir]'] = 'res/%s/' % dlc_case
mt['[log_dir]'] = 'logfiles/%s/' % dlc_case
mt['[htc_dir]'] = 'htc/%s/' % dlc_case
mt['[case_id]'] = mt['[Case id.]']
mt['[time_stop]'] = mt['[time stop]']
mt['[turb_base_name]'] = mt['[Turb base name]']
mt['[DLC]'] = mt['[Case id.]'].split('_')[0][3:]
mt['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
mt['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
mt['[iter_dir]'] = 'iter/%s/' % dlc_case
if mt['[eigen_analysis]']:
rpl = (dlc_case, mt['[Case id.]'])
mt['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
mt['[duration]'] = str(float(mt['[time_stop]']) - float(mt['[t0]']))
# replace nan with empty
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
if jj == 'nan':
mt[ii] = ''
return master
def tags_dlcs(master):
"""
Initiate tags that are defined in the DLC spreadsheets
"""
master.tags['[t0]'] = 0
master.tags['[time stop]'] = 0
master.tags['[Case folder]'] = 'test'
master.tags['[Case id.]'] = 'test'
master.tags['[Windspeed]'] = 8
master.tags['[wdir]'] = 0 # used for the user defined wind
master.tags['[wdir_rot]'] = 0 # used for the windfield rotations
master.tags['[tu_seed]'] = 0
master.tags['[tu_model]'] = 0
master.tags['[TI]'] = 0
master.tags['[Turb base name]'] = 'none'
master.tags['[turb_dx]'] = 1.0
master.tags['[shear_exp]'] = 0.2
master.tags['[wsp factor]'] = 1.0
master.tags['[gust]'] = False
master.tags['[gust_type]'] = ''
master.tags['[G_A]'] = ''
master.tags['[G_phi0]'] = ''
master.tags['[G_t0]'] = ''
master.tags['[G_T]'] = ''
master.tags['[Rotor azimuth]'] = 0
master.tags['[Free shaft rot]'] = ''
master.tags['[init_wr]'] = 0.5
master.tags['[Pitch 1 DLC22b]'] = 0
master.tags['[Rotor locked]'] = False
master.tags['[Time stuck DLC22b]'] = -1
master.tags['[Cut-in time]'] = -1
master.tags['[Cut-out time]'] = -1
master.tags['[Stop type]'] = -1
master.tags['[Pitvel 1]'] = 4
master.tags['[Pitvel 2]'] = 6
master.tags['[Grid loss time]'] = 1000
master.tags['[out_format]'] = 'hawc_binary'
master.tags['[Time pitch runaway]'] = 1000
master.tags['[Induction]'] = 1
master.tags['[Dyn stall]'] = 1
return master
def tags_defaults(master):
# other required tags and their defaults
master.tags['[dt_sim]'] = 0.02
master.tags['[hawc2_exe]'] = 'hawc2-latest'
# folder names for the saved results, htc, data, zip files
# Following dirs are relative to the model_dir_server and they specify
# the location of where the results, logfiles, animation files that where
# run on the server should be copied to after the simulation has finished.
# on the node, it will try to copy the turbulence files from these dirs
master.tags['[animation_dir]'] = 'animation/'
master.tags['[control_dir]'] = 'control/'
master.tags['[data_dir]'] = 'data/'
master.tags['[eigen_analysis]'] = False
master.tags['[eigenfreq_dir]'] = False
master.tags['[htc_dir]'] = 'htc/'
master.tags['[log_dir]'] = 'logfiles/'
master.tags['[meander_dir]'] = False
master.tags['[opt_dir]'] = False
master.tags['[pbs_out_dir]'] = 'pbs_out/'
master.tags['[res_dir]'] = 'res/'
master.tags['[iter_dir]'] = 'iter/'
master.tags['[turb_dir]'] = 'turb/'
master.tags['[turb_db_dir]'] = '../turb/'
master.tags['[wake_dir]'] = False
master.tags['[hydro_dir]'] = False
master.tags['[mooring_dir]'] = False
master.tags['[externalforce]'] = False
# zip_root_files only is used when copy to run_dir and zip creation, define
# in the HtcMaster object
master.tags['[zip_root_files]'] = []
# only active on PBS level, so files have to be present in the run_dir
master.tags['[copyback_files]'] = [] # copyback_resultfile
master.tags['[copyback_frename]'] = [] # copyback_resultrename
master.tags['[copyto_files]'] = [] # copyto_inputfile
master.tags['[copyto_generic]'] = [] # copyto_input_required_defaultname
master.tags['[eigen_analysis]'] = False
# =========================================================================
# basic required tags by HtcMaster and PBS in order to function properly
# =========================================================================
# the express queue ('#PBS -q xpresq') has a maximum walltime of 1h
master.tags['[pbs_queue_command]'] = '#PBS -q workq'
# walltime should have following format: hh:mm:ss
master.tags['[walltime]'] = '04:00:00'
master.tags['[auto_walltime]'] = False
return master
def excel_stabcon(proot, fext='xlsx', pignore=None, sheet=0,
pinclude=None):
"""
Read all MS Excel files that hold load case definitions according to
the team STABCON definitions. Save each case in a list according to the
opt_tags principles as used in Simulations.launch(). This method assumes
that a standard HAWC2 folder layout is used with the following folder
names: res, logfiles, htc, pbs_out, pbs_in, iter. Further some tags
are added to be compatible with the tag convention in the Simulations
module.
Parameters
----------
proot : string
Path that will be searched recursively for Excel files containing
load case definitions.
fext : string, default='xlsx'
File extension of the Excel files that should be loaded
pignore : string, default=None
Specify which string can not occur in the full path of the DLC target.
pinclude : string, default=None
Specify which string has to occur in the full path of the DLC target.
sheet : string or int, default=0
Name or index of the Excel sheet to be considered. By default, the
first sheet (index=0) is taken.
"""
print('looking for DLC spreadsheet definitions at:')
print(proot)
df_list = misc.read_excel_files(proot, fext=fext, pignore=pignore,
sheet=sheet, pinclude=pinclude)
print('found %i Excel file(s), ' % len(df_list), end='')
k = 0
for df in df_list:
k += len(df)
print('in which a total of %s cases are defined.' % k)
opt_tags = []
for (dlc, df) in viewitems(df_list):
# replace ';' with False, and Nan(='') with True
# this is more easy when testing for the presence of stuff compared
# to checking if a value is either True/False or ''/';'
# this doesn't work, it will result in 1 for True and 0 for False
# because the nan values have np.float dtype
# df.fillna(' ', inplace=True)
# df.replace(';', False, inplace=True)
# instead, convert everything to strings, this will maintain some nans
# as empty strings, but not all of them!
df2 = df.astype(str)
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.iteritems():
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
if isinstance(tags_dict[str(key)], str):
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
tags_dict['[Case folder]'] = tags_dict['[Case folder]'].lower()
tags_dict['[Case id.]'] = tags_dict['[Case id.]'].lower()
dlc_case = tags_dict['[Case folder]']
tags_dict['[data_dir]'] = 'data/'
tags_dict['[res_dir]'] = 'res/%s/' % dlc_case
tags_dict['[log_dir]'] = 'logfiles/%s/' % dlc_case
tags_dict['[htc_dir]'] = 'htc/%s/' % dlc_case
if '[Case id.]' in tags_dict.keys():
tags_dict['[case_id]'] = tags_dict['[Case id.]']
if '[time stop]' in tags_dict.keys():
tags_dict['[time_stop]'] = tags_dict['[time stop]']
try:
tags_dict['[turb_base_name]'] = tags_dict['[Turb base name]']
except KeyError:
tags_dict['[turb_base_name]'] = None
tags_dict['[Turb base name]'] = None
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
tags_dict['[DLC]'] = tags_dict['[Case id.]'].split('_')[0][3:]
tags_dict['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
tags_dict['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
tags_dict['[iter_dir]'] = 'iter/%s/' % dlc_case
# the default spreadsheets do not define the tags related to the
# eigen analsyis yet
if '[eigen_analysis]' in tags_dict and tags_dict['[eigen_analysis]']:
rpl = (dlc_case, tags_dict['[Case id.]'])
if '[eigenfreq_dir]' in tags_dict:
tags_dict['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
t_stop = float(tags_dict['[time_stop]'])
t0 = float(tags_dict['[t0]'])
tags_dict['[duration]'] = str(t_stop - t0)
opt_tags.append(tags_dict.copy())
return opt_tags
def read_tags_spreadsheet(fname):
"""Read a spreadsheet with HAWC2 tags, make sure no 0/1/nan ends up
replacing the ";" or "" (empty). Do not add any other tags.
Returns
-------
opt_tags : [{}, {}] list of dictionaries
"""
df = pd.read_excel(fname)
df2 = df.astype(str)
opt_tags = []
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.items():
if isinstance(value, str):
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
opt_tags.append(tags_dict.copy())
return opt_tags
class Tests(unittest.TestCase):
"""
"""
def setUp(self):
self.fpath = 'data/DLCs'
def test_read_tag_exchange_file(self):
df_list = misc.read_excel_files(self.fpath, fext='xlsx', pignore=None,
sheet=0, pinclude=None)
df = df_list[list(df_list.keys())[0]]