Newer
Older
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 5 14:01:25 2014
@author: dave
"""
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from builtins import str
from future import standard_library
standard_library.install_aliases()
import os
import unittest
import pandas as pd
from wetb.prepost import misc
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def casedict2xlsx():
"""
Convert a full Cases.cases dict to Excel spreadsheets
"""
def configure_dirs(verbose=False):
"""
Automatically configure required directories to launch simulations
"""
P_RUN = os.getcwd()
p_run_root = os.sep.join(P_RUN.split(os.sep)[:-2])
# MODEL SOURCES, exchanche file sources
P_SOURCE = P_RUN
# Project name, sim_id: derive from folder name
PROJECT = P_RUN.split(os.sep)[-2]
sim_id = P_RUN.split(os.sep)[-1]
master = find_master_file(P_SOURCE)
if master is None:
raise ValueError('Could not find master file in htc/_master')
MASTERFILE = master
P_MASTERFILE = os.path.join(P_SOURCE, 'htc%s_master%s' % (os.sep, os.sep))
POST_DIR = os.path.join(p_run_root, PROJECT, 'python-prepost-data%s' % os.sep)
if verbose:
print('='*79)
print('POST_DIR: %s' % POST_DIR)
print(' P_RUN: %s' % P_RUN)
print('P_SOURCE: %s' % P_SOURCE)
print(' PROJECT: %s' % PROJECT)
print(' sim_id: %s' % sim_id)
print(' master: %s' % MASTERFILE)
print('='*79)
return P_RUN, P_SOURCE, PROJECT, sim_id, P_MASTERFILE, MASTERFILE, POST_DIR
def find_master_file(proot, htc_dir='htc', master_dir='_master',
master_contains='_master_'):
"""
Find the master file name. It is assumed that the master file is in the
folder _master, under htc, and contains _master_ in the file name.
"""
for root, dirs, files in os.walk(os.path.join(proot, htc_dir, master_dir)):
for fname in files:
if fname.find(master_contains) > -1:
return fname
return None
def variable_tag_func(master, case_id_short=False):
"""
When using the Excel definitions, and the whole default setup, the
variable_tag_func is not required to do anything extra.
"""
# -------------------------------------------------------------------------
# mt = master
# V = mt['windspeed']
# mt['duration'] = mt['time_stop'] - mt['t0']
# t = mt['duration']
# if V > abs(1e-15):
# b = 5.6
# mt['TI'] = mt['TI_ref'] * ((0.75*V) + b) / V # NTM
# # ETM
# c = 2.0
# V_ave = 0.2 * 50.0
# sigma = mt['TI_ref'] / V
# mt['TI'] = sigma * c * (0.072 * (V_ave / c + 3.0) * (V / c - 4.0) + 10.0)
# else:
# mt['TI'] = 0
#
# mt['turb_dx'] = V*t/mt['turb_grid_x']
#
# mt['turb_dy'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# mt['turb_dz'] = (mt['rotor_diameter'] / mt['turb_grid_yz'])*1.1
#
# # check: dx spacing should be 0.1*mean_windspeed and 0.2*mean_windspeed
# # between 0.1 and 0.2 seconds between points
# if not (V*0.1 < mt['turb_dx'] < V*0.2):
# logging.warn('turbulence spacing dx out of bounds')
# print('%5.3f %5.3f %5.3f' % (V*0.1, mt['turb_dx'], V*0.2))
#
# #mt['turb_base_name'] = 'turb_s' + str(mt['turb_seed']) + '_' + str(V)
# mt['turb_base_name'] = 'turb_s%i_%1.2f' % (mt['turb_seed'], V)
# -------------------------------------------------------------------------
return master
def vartags_dlcs(master):
mt = master.tags
dlc_case = mt['[Case folder]']
mt['[data_dir]'] = 'data/'
mt['[res_dir]'] = 'res/%s/' % dlc_case
mt['[log_dir]'] = 'logfiles/%s/' % dlc_case
mt['[htc_dir]'] = 'htc/%s/' % dlc_case
mt['[case_id]'] = mt['[Case id.]']
mt['[time_stop]'] = mt['[time stop]']
mt['[turb_base_name]'] = mt['[Turb base name]']
mt['[DLC]'] = mt['[Case id.]'].split('_')[0][3:]
mt['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
mt['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
mt['[iter_dir]'] = 'iter/%s/' % dlc_case
if mt['[eigen_analysis]']:
rpl = (dlc_case, mt['[Case id.]'])
mt['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
mt['[duration]'] = str(float(mt['[time_stop]']) - float(mt['[t0]']))
# replace nan with empty
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
if jj == 'nan':
mt[ii] = ''
return master
def tags_dlcs(master):
"""
Initiate tags that are defined in the DLC spreadsheets
"""
master.tags['[t0]'] = 0
master.tags['[time stop]'] = 0
master.tags['[Case folder]'] = 'test'
master.tags['[Case id.]'] = 'test'
master.tags['[Windspeed]'] = 8
master.tags['[wdir]'] = 0 # used for the user defined wind
master.tags['[wdir_rot]'] = 0 # used for the windfield rotations
master.tags['[tu_seed]'] = 0
master.tags['[tu_model]'] = 0
master.tags['[TI]'] = 0
master.tags['[Turb base name]'] = 'none'
master.tags['[turb_dx]'] = 1.0
master.tags['[shear_exp]'] = 0.2
master.tags['[wsp factor]'] = 1.0
master.tags['[gust]'] = False
master.tags['[gust_type]'] = ''
master.tags['[G_A]'] = ''
master.tags['[G_phi0]'] = ''
master.tags['[G_t0]'] = ''
master.tags['[G_T]'] = ''
master.tags['[Rotor azimuth]'] = 0
master.tags['[Free shaft rot]'] = ''
master.tags['[init_wr]'] = 0.5
master.tags['[Pitch 1 DLC22b]'] = 0
master.tags['[Rotor locked]'] = False
master.tags['[Time stuck DLC22b]'] = -1
master.tags['[Cut-in time]'] = -1
master.tags['[Cut-out time]'] = -1
master.tags['[Stop type]'] = -1
master.tags['[Pitvel 1]'] = 4
master.tags['[Pitvel 2]'] = 6
master.tags['[Grid loss time]'] = 1000
master.tags['[out_format]'] = 'hawc_binary'
master.tags['[Time pitch runaway]'] = 1000
master.tags['[Induction]'] = 1
master.tags['[Dyn stall]'] = 1
return master
def tags_defaults(master):
# other required tags and their defaults
master.tags['[dt_sim]'] = 0.02
master.tags['[hawc2_exe]'] = 'hawc2-latest'
# folder names for the saved results, htc, data, zip files
# Following dirs are relative to the model_dir_server and they specify
# the location of where the results, logfiles, animation files that where
# run on the server should be copied to after the simulation has finished.
# on the node, it will try to copy the turbulence files from these dirs
master.tags['[animation_dir]'] = 'animation/'
master.tags['[control_dir]'] = 'control/'
master.tags['[data_dir]'] = 'data/'
master.tags['[eigen_analysis]'] = False
master.tags['[eigenfreq_dir]'] = False
master.tags['[htc_dir]'] = 'htc/'
master.tags['[log_dir]'] = 'logfiles/'
master.tags['[meander_dir]'] = False
master.tags['[opt_dir]'] = False
master.tags['[pbs_out_dir]'] = 'pbs_out/'
master.tags['[res_dir]'] = 'res/'
master.tags['[iter_dir]'] = 'iter/'
master.tags['[turb_dir]'] = 'turb/'
master.tags['[turb_db_dir]'] = '../turb/'
master.tags['[wake_dir]'] = False
master.tags['[hydro_dir]'] = False
master.tags['[mooring_dir]'] = False
master.tags['[externalforce]'] = False
# zip_root_files only is used when copy to run_dir and zip creation, define
# in the HtcMaster object
master.tags['[zip_root_files]'] = []
# only active on PBS level, so files have to be present in the run_dir
master.tags['[copyback_files]'] = [] # copyback_resultfile
master.tags['[copyback_frename]'] = [] # copyback_resultrename
master.tags['[copyto_files]'] = [] # copyto_inputfile
master.tags['[copyto_generic]'] = [] # copyto_input_required_defaultname
master.tags['[eigen_analysis]'] = False
# =========================================================================
# basic required tags by HtcMaster and PBS in order to function properly
# =========================================================================
# the express queue ('#PBS -q xpresq') has a maximum walltime of 1h
master.tags['[pbs_queue_command]'] = '#PBS -q workq'
# walltime should have following format: hh:mm:ss
master.tags['[walltime]'] = '04:00:00'
master.tags['[auto_walltime]'] = False
return master
def excel_stabcon(proot, fext='xlsx', pignore=None, sheet=0,
pinclude=None):
"""
Read all MS Excel files that hold load case definitions according to
the team STABCON definitions. Save each case in a list according to the
opt_tags principles as used in Simulations.launch(). This method assumes
that a standard HAWC2 folder layout is used with the following folder
names: res, logfiles, htc, pbs_out, pbs_in, iter. Further some tags
are added to be compatible with the tag convention in the Simulations
module.
Parameters
----------
proot : string
Path that will be searched recursively for Excel files containing
load case definitions.
fext : string, default='xlsx'
File extension of the Excel files that should be loaded
pignore : string, default=None
Specify which string can not occur in the full path of the DLC target.
pinclude : string, default=None
Specify which string has to occur in the full path of the DLC target.
sheet : string or int, default=0
Name or index of the Excel sheet to be considered. By default, the
first sheet (index=0) is taken.
"""
print('looking for DLC spreadsheet definitions at:')
print(proot)
df_list = misc.read_excel_files(proot, fext=fext, pignore=pignore,
sheet=sheet, pinclude=pinclude)
print('found %i Excel file(s), ' % len(df_list), end='')
k = 0
for df in df_list:
k += len(df)
print('in which a total of %s cases are defined.' % k)
opt_tags = []
for dlc, df in df_list.items():
# replace ';' with False, and Nan(='') with True
# this is more easy when testing for the presence of stuff compared
# to checking if a value is either True/False or ''/';'
# this doesn't work, it will result in 1 for True and 0 for False
# because the nan values have np.float dtype
# df.fillna(' ', inplace=True)
# df.replace(';', False, inplace=True)
# instead, convert everything to strings, this will maintain some nans
# as empty strings, but not all of them!
df2 = df.astype(str)
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.items():
if isinstance(value, str):
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
# if isinstance(tags_dict[str(key)], str):
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
tags_dict['[Case folder]'] = tags_dict['[Case folder]'].lower()
tags_dict['[Case id.]'] = tags_dict['[Case id.]'].lower()
dlc_case = tags_dict['[Case folder]']
tags_dict['[data_dir]'] = 'data/'
tags_dict['[res_dir]'] = 'res/%s/' % dlc_case
tags_dict['[log_dir]'] = 'logfiles/%s/' % dlc_case
tags_dict['[htc_dir]'] = 'htc/%s/' % dlc_case
tags_dict['[case_id]'] = tags_dict['[Case id.]']
tags_dict['[time_stop]'] = tags_dict['[time stop]']
tags_dict['[turb_base_name]'] = tags_dict['[Turb base name]']
tags_dict['[DLC]'] = tags_dict['[Case id.]'].split('_')[0][3:]
tags_dict['[pbs_out_dir]'] = 'pbs_out/%s/' % dlc_case
tags_dict['[pbs_in_dir]'] = 'pbs_in/%s/' % dlc_case
tags_dict['[iter_dir]'] = 'iter/%s/' % dlc_case
# the default spreadsheets do not define the tags related to the
# eigen analsyis yet
if '[eigen_analysis]' in tags_dict and tags_dict['[eigen_analysis]']:
rpl = (dlc_case, tags_dict['[Case id.]'])
if '[eigenfreq_dir]' in tags_dict:
tags_dict['[eigenfreq_dir]'] = 'res_eigen/%s/%s/' % rpl
t_stop = float(tags_dict['[time_stop]'])
t0 = float(tags_dict['[t0]'])
tags_dict['[duration]'] = str(t_stop - t0)
opt_tags.append(tags_dict.copy())
return opt_tags
def read_tags_spreadsheet(fname):
"""Read a spreadsheet with HAWC2 tags, make sure no 0/1/nan ends up
replacing the ";" or "" (empty). Do not add any other tags.
Returns
-------
opt_tags : [{}, {}] list of dictionaries
"""
df = pd.read_excel(fname)
df2 = df.astype(str)
opt_tags = []
for count, row in df2.iterrows():
tags_dict = {}
# construct to dict, convert unicode keys/values to strings
for key, value in row.items():
if isinstance(value, str):
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
tags_dict[str(key)] = str(value)
else:
tags_dict[str(key)] = value
# convert ; and empty to False/True
if tags_dict[str(key)] == ';':
tags_dict[str(key)] = False
elif tags_dict[str(key)] == '':
tags_dict[str(key)] = True
elif tags_dict[str(key)].lower() == 'nan':
tags_dict[str(key)] = True
opt_tags.append(tags_dict.copy())
return opt_tags
class Tests(unittest.TestCase):
"""
"""
def setUp(self):
self.fpath = 'data/DLCs'
def test_read_tag_exchange_file(self):
df_list = misc.read_excel_files(self.fpath, fext='xlsx', pignore=None,
sheet=0, pinclude=None)
df = df_list[list(df_list.keys())[0]]