diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4edd3b747cfb58bf25b1d4222b8bc13f06251095..a510aab648f45657254307a706f8cec5833869ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,9 @@
before_script:
- apt-get update
+ # uncomment first time
#- rm -rf TestFiles
- - git submodule sync
#- git submodule update --init
+ - git submodule sync
- git submodule update
diff --git a/docs/generate-spreadsheet.md b/docs/generate-spreadsheet.md
index ff87c08675324d45a2836853a08e5b1746dd0fa3..19fa7fd9dec4969f5e363adcc0c63240a42fe294 100644
--- a/docs/generate-spreadsheet.md
+++ b/docs/generate-spreadsheet.md
@@ -15,9 +15,9 @@ This tool comes handy in the following scenarios:
The generator of the cases uses an input spreadsheet where the cases are defined
in a more compact way.
-The tool is based on the "tags" concept that is used for the genetaion of the htc files.
+The tool is based on the "tags" concept that is used for the generation of the htc files.
-Main spreatsheet
+Main spreadsheet
----------------
A main spreadsheet is used to defines all the DLC of the DLB. The file specifies the tags that are then required in the htc files.
@@ -26,13 +26,38 @@ The file has:
* a Main sheet where some wind turbines parameters are defined, the tags are initialized, and the definitions of turbulence and gusts are given.
* a series of other sheets, each defining a DLC. In these sheets the tags that changes in that DLC are defined.
-The tags are devided into three possible different categories:
+The tags are divided into three possible different categories:
* Constants (C). Constants are tags that do not change in a DLC, e.g. simulation time, output format, ...;
-* Variables (V). Variables are tags that define the number of cases in a DLC through their combinations, e.g. wind speed, number of turbilence seeds, wind direction, ..;
+* Variables (V). Variables are tags that define the number of cases in a DLC through their combinations, e.g. wind speed, number of turbulence seeds, wind direction, ..;
* Functions (F). Functions are tags that depend on other tags through an expression, e.g. turbulence intensity, case name, ....
In each sheet the type of tag is defined in the line above the tag by typing one of the letters C, V, or F.
+Functions (F) tags
+------------------
+
+* Numbers can be converted to strings (for example when a tag refers to a file name)
+by using double quotes ```"``` for Functions (F):
+ * ```"wdir_[wdir]deg_wsp_[wsp]ms"``` will result in the tags ``` [wdir]```
+ and ```[wsp]``` being replaced with formatted text.
+ * following formatting rules are used:
+ * ```[wsp]```, ```[gridgustdelay]``` : ```02i```
+ * ```[wdir]```, ```[G_phi0]``` : ```03i```
+ * ```[Hs]```, ```[Tp]``` : ```05.02f```
+ * all other tags: ```04i```
+ * Only numbers in tags with double quotes are formatted. In all other cases
+ there is no formatting taking place and hence no loss of precision occurs.
+ * In this context, when using quotes, always use double quotes like ```"```.
+ Do not use single quotes ```'``` or any other quote character.
+
+Variable (V) tags
+-----------------
+
+* ```[seed]``` and ```[wave_seed]``` are special variable tags. Instead of defining
+a range of seeds, the user indicates the number of seeds to be used.
+* ```[wsp]``` is a required variable tag
+* ```[seed]``` should be placed in a column BEFORE ```[wsp]```
+
Generate the files
------------------
diff --git a/docs/howto-make-dlcs.md b/docs/howto-make-dlcs.md
index a41d445dbfa084d76b09f07ff3654303e1ef39ed..144eed6ed3af387e2813cef951add8b35e474433 100644
--- a/docs/howto-make-dlcs.md
+++ b/docs/howto-make-dlcs.md
@@ -263,7 +263,7 @@ When there is a new version of HAWC2, or when a new license manager is released,
you can update your local wine directory as follows:
```
-g-000 $ cp /home/MET/hawc2exe/* /home/$USER/wine_exe/win32/
+g-000 $ rsync -au /home/MET/hawc2exe/win32 /home/$USER/wine_exe/win32 --progress
```
The file ```hawc2-latest.exe``` will always be the latest HAWC2
diff --git a/requirements.txt b/requirements.txt
index 889438e8f44a2119fc04764ae9a642812055c844..270f82f988f5d487fea5e0254aa94e285e2aedc4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
# Add your requirements here like:
six
+cython
numpy>=1.4
scipy>=0.9
matplotlib
diff --git a/wetb/prepost/GenerateDLCs.py b/wetb/prepost/GenerateDLCs.py
index f1cf530d20b06626d155cfcec8dc6c777d53e5a1..f4c83750964c2e12541a5b237bd2690e7a436740 100644
--- a/wetb/prepost/GenerateDLCs.py
+++ b/wetb/prepost/GenerateDLCs.py
@@ -79,32 +79,33 @@ class GeneralDLC(object):
cases_len.append(len(v))
cases_index = multi_for(list(map(range, cases_len)))
-# for irow, row in enumerate(cases_index):
-# counter = floor(irow/len(variables['[wsp]']))+1
-# for icol, col in enumerate(row):
-# if variables_order[icol] == '[seed]':
-# value = '%4.4i' % (1000*counter + row[variables_order.index('[wsp]')]+1)
-# elif variables_order[icol] == '[wave_seed]': #shfe: wave_seed
-# value = '%4.4i' % (1000*counter + row[variables_order.index('[wsp]')]+1)
-# else:
-# value = variables[variables_order[icol]][col]
-# if not isinstance(value, float) and not isinstance(value, int):
-# value = str(value)
-# dlc[variables_order[icol]].append(value)
+ # when no seeds are used, otherwise i_seed is not set
+ i_seed = -1
+ if '[wsp]' in variables_order:
+ i_wsp = variables_order.index('[wsp]')
+ len_wsp = len(variables['[wsp]'])
+ else:
+ raise ValueError('Missing VARIABLE (V) [wsp] tag!')
+ if '[seed]' in variables_order:
+ i_seed = variables_order.index('[seed]')
+ if '[wave_seed]' in variables_order:
+ i_wave_seed = variables_order.index('[wave_seed]')
+ if i_seed > i_wsp:
+ raise ValueError('column [seed] should come BEFORE [wsp] !!')
+
for irow, row in enumerate(cases_index):
- counter = floor(irow/len(variables['[wsp]']))+1
for icol, col in enumerate(row):
if variables_order[icol] == '[seed]':
- value = '%4.4i' % (1000*counter + row[variables_order.index('[wsp]')]+1)
+ counter = floor(irow/len_wsp) + 1
+ value = '%4.4i' % (1000*counter + row[i_wsp] + 1)
elif variables_order[icol] == '[wave_seed]':
- value = '%4.4i' % ( 100*(row[variables_order.index('[wsp]')]+1) + \
- row[variables_order.index('[wave_seed]')]+1)
-
+ value = '%4.4i' % (100*(row[i_wsp]+1) + row[i_wave_seed] + 1)
+# value = '%4.4i' % (1000*counter + row[i_wsp] + 101)
# value = '%4.4i' % (irow+1)
-# value = '%4.4i' % (10000*(row[variables_order.index('[wave_dir]')]+1) + \
-# 1000*(row[variables_order.index('[Hs]')]+1) + \
-# 10*(row[variables_order.index('[Tp]')]+1) +\
-# row[variables_order.index('[seed]')]+1)
+# value = '%4.4i' % (10000*(row[i_wave_dir])] + 1) + \
+# 1000*(row[i_Hs])] + 1) + \
+# 10*(row[i_Tp])] + 1) +\
+# row[i_seed])] + 1)
else:
value = variables[variables_order[icol]][col]
@@ -153,8 +154,8 @@ class GeneralDLC(object):
# specify the precision of the tag as used in the formulas
# this does NOT affect the precision of the tag itself, only when used
# in a formula based tag.
- formats = {'[wsp]':'%2.2i', '[gridgustdelay]':'%2.2i',
- '[wdir]':'%3.3i', '[G_phi0]':'%3.3i',
+ formats = {'[wsp]':'%02i', '[gridgustdelay]':'%02i',
+ '[wdir]':'%03i', '[G_phi0]':'%03i',
'[sign]':'%s',
'[Hs]':'%05.02f', '[Tp]':'%05.02f'}
@@ -168,7 +169,7 @@ class GeneralDLC(object):
try:
fmt = formats[key]
except KeyError:
- fmt = '%4.4i'
+ fmt = '%04i'
try:
value = float(dlc[key][i])
except ValueError:
@@ -209,14 +210,15 @@ class GenerateDLCCases(GeneralDLC):
"""
- def execute(self, filename='DLCs.xlsx', folder=''):
+ def execute(self, filename='DLCs.xlsx', folder='', isheets=None):
book = xlrd.open_workbook(filename)
- nsheets = book.nsheets
+ if isheets is None:
+ isheets = list(range(1, book.nsheets))
# Loop through all the sheets. Each sheet correspond to a DLC.
- for isheet in range(1, nsheets):
+ for isheet in isheets:
# Read all the initialization constants and functions in the
# first sheet
@@ -270,6 +272,7 @@ class GenerateDLCCases(GeneralDLC):
self.add_constants_tag(dlc, constants)
self.add_formulas(dlc, formulas)
self.add_formulas(dlc, general_functions)
+ # TODO: before eval, check if all tags in formula's are present
self.eval_formulas(dlc)
df = pd.DataFrame(dlc)
if not os.path.exists(folder):
@@ -277,29 +280,6 @@ class GenerateDLCCases(GeneralDLC):
df.to_excel(os.path.join(folder, sheet.name+'.xlsx'), index=False)
-class RunTest():
- """
- Class to perform basic testing of the GenerateDLCCases class. It writes the
- spreadsheets and compare them with a reference set.
- """
- def execute(self):
-
- from pandas.util.testing import assert_frame_equal
- a = GenerateDLCCases()
- a.execute()
-
- book = xlrd.open_workbook('DLCs.xlsx')
- nsheets = book.nsheets
- for isheet in range(1, nsheets):
- sheet = book.sheets()[isheet]
- print('Sheet #%i' % isheet, sheet.name)
- book1 = pd.read_excel('Reference/'+sheet.name+'.xlsx')
-
- book2 = pd.read_excel(sheet.name+'.xls')
-
- book2 = book2[book1.columns]
- assert_frame_equal(book1, book2, check_dtype=False)
-
if __name__ == '__main__':
parser = ArgumentParser(description = "generator of DLB spreadsheets")
diff --git a/wetb/prepost/dlcplots.py b/wetb/prepost/dlcplots.py
index 7074cdda41a059ad55bd6675549cc31a973f1db1..21628cda9e37c17f419ca5acc03a526b9a58c256 100644
--- a/wetb/prepost/dlcplots.py
+++ b/wetb/prepost/dlcplots.py
@@ -21,7 +21,7 @@ import gc
import numpy as np
import matplotlib.pyplot as plt
-import matplotlib as mpl
+#import matplotlib as mpl
#from matplotlib.figure import Figure
#from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigCanvas
#from scipy import interpolate as interp
@@ -45,8 +45,8 @@ plt.rc('xtick', labelsize=10)
plt.rc('ytick', labelsize=10)
plt.rc('axes', labelsize=12)
# do not use tex on Gorm and or Jess
-if not socket.gethostname()[:2] in ['g-', 'je', 'j-']:
- plt.rc('text', usetex=True)
+#if not socket.gethostname()[:2] in ['g-', 'je', 'j-']:
+# plt.rc('text', usetex=True)
plt.rc('legend', fontsize=11)
plt.rc('legend', numpoints=1)
plt.rc('legend', borderaxespad=0)
@@ -55,6 +55,9 @@ plt.rc('legend', borderaxespad=0)
def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False):
"""
"""
+
+ cols_extra = ['[run_dir]', '[res_dir]', '[wdir]', '[DLC]', '[Case folder]']
+
# map the run_dir to the same order as the post_dirs, labels
run_dirs = []
# avoid saving merged cases if there is only one!
@@ -80,13 +83,18 @@ def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False):
else:
wsp = '[Windspeed]'
# columns we want to add from cc.cases (cases dict) to stats
- cols_cc = set(['[run_dir]', wsp, '[res_dir]', '[wdir]', '[DLC]'])
+ cols_cc = set(cols_extra + [wsp])
# do not add column twice, some might already be in df stats
add_cols = list(cols_cc - set(df_stats.columns))
add_cols.append('[case_id]')
dfc = dfc[add_cols]
df_stats = pd.merge(df_stats, dfc, on='[case_id]')
- df_stats.rename(columns={wsp:'[Windspeed]'}, inplace=True)
+ # FIXME: this is very messy, we can end up with both [wsp] and
+ # [Windspeed] columns
+ if '[Windspeed]' in df_stats.columns and '[wsp]' in df_stats.columns:
+ df_stats.drop('[wsp]', axis=1, inplace=True)
+ if wsp != '[Windspeed]':
+ df_stats.rename(columns={wsp:'[Windspeed]'}, inplace=True)
# map the run_dir to the same order as the post_dirs, labels
run_dirs.append(df_stats['[run_dir]'].unique()[0])
@@ -120,8 +128,8 @@ def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False):
del df_stats, _, cc
gc.collect()
# and load the reduced combined set
- print('loading merged stats: %s' % fpath)
- df_stats = pd.read_hdf(fpath, 'table')
+ print('loading merged stats: %s' % fmerged)
+ df_stats = pd.read_hdf(fmerged, 'table')
else:
sim_id = sim_ids
sim_ids = [sim_id]
@@ -135,18 +143,21 @@ def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False):
# stats has only a few columns identifying the different cases
# add some more for selecting them
dfc = cc.cases2df()
- if 'wsp' in dfc.columns:
+ if '[wsp]' in dfc.columns:
wsp = '[wsp]'
else:
wsp = '[Windspeed]'
# columns we want to add from cc.cases (cases dict) to stats
- cols_cc = set(['[run_dir]', wsp, '[res_dir]', '[wdir]', '[DLC]'])
+ cols_cc = set(cols_extra + [wsp])
# do not add column twice, some might already be in df stats
add_cols = list(cols_cc - set(df_stats.columns))
add_cols.append('[case_id]')
dfc = dfc[add_cols]
df_stats = pd.merge(df_stats, dfc, on='[case_id]')
- df_stats.rename(columns={wsp:'[Windspeed]'}, inplace=True)
+ if '[Windspeed]' in df_stats.columns and '[wsp]' in df_stats.columns:
+ df_stats.drop('[wsp]', axis=1, inplace=True)
+ if wsp != '[Windspeed]':
+ df_stats.rename(columns={wsp:'[Windspeed]'}, inplace=True)
return run_dirs, df_stats
@@ -155,25 +166,27 @@ def merge_sim_ids(sim_ids, post_dirs, post_dir_save=False):
# =============================================================================
def plot_stats2(sim_ids, post_dirs, plot_chans, fig_dir_base=None, labels=None,
- post_dir_save=False, dlc_ignore=['00'], figsize=(8,6)):
+ post_dir_save=False, dlc_ignore=['00'], figsize=(8,6),
+ eps=False, ylabels=None):
"""
Map which channels have to be compared
"""
# reduce required memory, only use following columns
cols = ['[run_dir]', '[DLC]', 'channel', '[res_dir]', '[Windspeed]',
- 'mean', 'max', 'min', 'std', '[wdir]']
+ 'mean', 'max', 'min', 'std', '[wdir]', '[Case folder]']
run_dirs, df_stats = merge_sim_ids(sim_ids, post_dirs,
post_dir_save=post_dir_save)
plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=labels,
- figsize=figsize, dlc_ignore=dlc_ignore)
+ figsize=figsize, dlc_ignore=dlc_ignore, eps=eps,
+ ylabels=ylabels)
def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
figsize=(8,6), dlc_ignore=['00'], run_dirs=None,
- sim_ids=[]):
+ sim_ids=[], eps=False, ylabels=None):
"""Create for each DLC an overview plot of the statistics.
df_stats required columns:
@@ -205,7 +218,7 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
figsize : tuple, default=(8,6)
- dlc_ignore : list, default=['00']
+ dlc_ignore : list, default=['dlc00']
By default all but dlc00 (stair case, wind ramp) are plotted. Add
more dlc numbers here if necessary.
@@ -225,7 +238,8 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
mfcs3 = ['r', 'w']
stds = ['r', 'b']
- required = ['[DLC]', '[run_dir]', '[wdir]', '[Windspeed]', '[res_dir]']
+ required = ['[DLC]', '[run_dir]', '[wdir]', '[Windspeed]', '[res_dir]',
+ '[Case folder]']
cols = df_stats.columns
for col in required:
if col not in cols:
@@ -243,9 +257,15 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
sim_ids.append(run_dir.split(os.path.sep)[-2])
# first, take each DLC appart
- for dlc_name, gr_dlc in df_stats.groupby(df_stats['[DLC]']):
+ for gr_name, gr_dlc in df_stats.groupby(df_stats['[Case folder]']):
+ dlc_name = gr_name
+ if dlc_name[:3].lower() == 'dlc':
+ # FIXME: this is messy since this places a hard coded dependency
+ # between [Case folder] and [Case id.] when the tag [DLC] is
+ # defined in dlcdefs.py
+ dlc_name = gr_name.split('_')[0]
# do not plot the stats for dlc00
- if dlc_name in dlc_ignore:
+ if dlc_name.lower() in dlc_ignore:
continue
# cycle through all the target plot channels
for ch_dscr, ch_names in plot_chans.items():
@@ -253,8 +273,9 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
# identical, we need to manually pick them.
# figure file name will be the first channel
if isinstance(ch_names, list):
+ ch_name = ch_names[0]
df_chan = gr_dlc[gr_dlc.channel == ch_names[0]]
- fname_base = ch_names[0].replace(' ', '_')
+ fname_base = ch_names[0]#.replace(' ', '_')
try:
df2 = gr_dlc[gr_dlc.channel == ch_names[1]]
df_chan = pd.concat([df_chan, df2], ignore_index=True)
@@ -262,9 +283,9 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
pass
else:
ch_name = ch_names
- ch_names = [ch_name]
+ ch_names = [ch_names]
df_chan = gr_dlc[gr_dlc.channel == ch_names]
- fname_base = ch_names.replace(' ', '_')
+ fname_base = ch_names#.replace(' ', '_')
# if not, than we are missing a channel description, or the channel
# is simply not available in the given result set
@@ -288,7 +309,7 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
elif len(ch_names) > 1 and len(lens)==2 and lens[1] < 1:
continue
- print('start plotting: %s %s' % (str(dlc_name).ljust(7), ch_dscr))
+ print('start plotting: %s %s' % (dlc_name.ljust(10), ch_dscr))
fig, axes = mplutils.make_fig(nrows=1, ncols=1,
figsize=figsize, dpi=120)
@@ -318,8 +339,10 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
# for clarity, set off-set on wind speed when comparing two DLB's
if len(lens)==2:
windoffset = [-0.2, 0.2]
+ dirroffset = [-5, 5]
else:
windoffset = [0]
+ dirroffset = [0]
# in case of a fully empty plot xlims will remain None and there
# is no need to save the plot
xlims = None
@@ -338,8 +361,8 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
# sid_names.append(sid_name)
print(' sim_id/label:', sid_name)
# FIXME: will this go wrong in PY3?
- if str(dlc_name) in ['61', '62']:
- xdata = gr_ch_dlc_sid['[wdir]'].values
+ if dlc_name.lower() in ['dlc61', 'dlc62']:
+ xdata = gr_ch_dlc_sid['[wdir]'].values + dirroffset[ii]
xlabel = 'wind direction [deg]'
xlims = [0, 360]
else:
@@ -354,12 +377,12 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
lab1 = 'mean'
lab2 = 'min'
lab3 = 'max'
- lab4 = 'std'
+# lab4 = 'std'
else:
lab1 = 'mean %s' % sid_name
lab2 = 'min %s' % sid_name
lab3 = 'max %s' % sid_name
- lab4 = 'std %s' % sid_name
+# lab4 = 'std %s' % sid_name
mfc1 = mfcs1[ii]
mfc2 = mfcs2[ii]
mfc3 = mfcs3[ii]
@@ -393,13 +416,18 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
ax.grid()
ax.set_xlim(xlims)
- leg = ax.legend(loc='best', ncol=3)
+ leg = ax.legend(bbox_to_anchor=(1, 1), loc='lower right', ncol=3)
leg.get_frame().set_alpha(0.7)
- ax.set_title(r'{DLC%s} %s' % (dlc_name, ch_dscr))
+# ax.set_title(r'{%s} %s' % (dlc_name.replace('_', '\\_'), ch_dscr))
+# fig.suptitle(r'{%s} %s' % (dlc_name.replace('_', '\\_'), ch_dscr))
+ fig.suptitle('%s %s' % (dlc_name, ch_dscr))
ax.set_xlabel(xlabel)
+ if ylabels is not None:
+ ax.set_ylabel(ylabels[ch_name])
fig.tight_layout()
- fig.subplots_adjust(top=0.92)
- fig_path = os.path.join(fig_dir, 'dlc%s' % dlc_name)
+ spacing = 0.92 - (0.065 * (ii + 1))
+ fig.subplots_adjust(top=spacing)
+ fig_path = os.path.join(fig_dir, dlc_name)
if len(sim_ids)==1:
fname = fname_base + '.png'
else:
@@ -408,6 +436,8 @@ def plot_dlc_stats(df_stats, plot_chans, fig_dir_base, labels=None,
os.makedirs(fig_path)
fig_path = os.path.join(fig_path, fname)
fig.savefig(fig_path)#.encode('latin-1')
+ if eps:
+ fig.savefig(fig_path.replace('.png', '.eps'))
fig.clear()
print('saved: %s' % fig_path)
diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
index ee86a5dd9f3d0241ced94e9df46bdde8370bf0ee..b86b4e26e0b02991960b068da96c16f66f00fab7 100644
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -265,7 +265,8 @@ def variable_tag_func_mod1(master, case_id_short=False):
def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
runmethod=None, write_htc=True, zipchunks=False,
- walltime='04:00:00', postpro_node=False):
+ walltime='04:00:00', postpro_node=False,
+ dlcs_dir='htc/DLCs'):
"""
Launch load cases defined in Excel files
"""
@@ -279,7 +280,6 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
pyenv = None
# see if a htc/DLCs dir exists
- dlcs_dir = os.path.join(P_SOURCE, 'htc', 'DLCs')
# Load all DLC definitions and make some assumptions on tags that are not
# defined
if os.path.exists(dlcs_dir):
@@ -458,11 +458,14 @@ def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True,
return df_stats, df_AEP, df_Leq
-def postpro_node_merge(tqdm=False):
+def postpro_node_merge(tqdm=False, zipchunks=False):
"""With postpro_node each individual case has a .csv file for the log file
analysis and a .csv file for the statistics tables. Merge all these single
files into one table/DataFrame.
+ When using the zipchunk approach, all log file analysis and statistics
+ are grouped into tar archives in the prepost-data directory.
+
Parameters
----------
@@ -470,12 +473,18 @@ def postpro_node_merge(tqdm=False):
Set to True for displaying a progress bar (provided by the tqdm module)
when merging all csv files into a single table/pd.DataFrame.
+ zipchunks : boolean, default=False
+ Set to True if merging post-processing files grouped into tar archives
+ as generated by the zipchunks approach.
+
"""
# -------------------------------------------------------------------------
# MERGE POSTPRO ON NODE APPROACH INTO ONE DataFrame
# -------------------------------------------------------------------------
lf = windIO.LogFile()
path_pattern = os.path.join(P_RUN, 'logfiles', '*', '*.csv')
+ if zipchunks:
+ path_pattern = os.path.join(POST_DIR, 'loganalysis_chnk*.tar.xz')
csv_fname = '%s_ErrorLogs.csv' % sim_id
fcsv = os.path.join(POST_DIR, csv_fname)
mdf = AppendDataFrames(tqdm=tqdm)
@@ -489,6 +498,8 @@ def postpro_node_merge(tqdm=False):
# -------------------------------------------------------------------------
path_pattern = os.path.join(P_RUN, 'res', '*', '*.csv')
csv_fname = '%s_statistics.csv' % sim_id
+ if zipchunks:
+ path_pattern = os.path.join(POST_DIR, 'statsdel_chnk*.tar.xz')
fcsv = os.path.join(POST_DIR, csv_fname)
mdf = AppendDataFrames(tqdm=tqdm)
# individual log file analysis does not have header, make sure to include
@@ -511,13 +522,22 @@ def postpro_node_merge(tqdm=False):
# -------------------------------------------------------------------------
# merge missing cols onto stats
required = ['[DLC]', '[run_dir]', '[wdir]', '[Windspeed]', '[res_dir]',
- '[case_id]']
+ '[case_id]', '[Case folder]']
df = pd.read_hdf(fdf, 'table')
+
+ # df now has case_id as the path to the statistics file: res/dlc12_xxx/yyy
+ # while df_tags will have just yyy as case_id
+ tmp = df['[case_id]'].str.split('/', expand=True)
+ df['[case_id]'] = tmp[tmp.columns[-1]]
+
cc = sim.Cases(POST_DIR, sim_id)
df_tags = cc.cases2df()[required]
df_stats = pd.merge(df, df_tags, on=['[case_id]'])
- df_stats.to_hdf(fdf, 'table')
- df_stats.to_csv(fdf.replace('.h5', '.csv'))
+ # if the merge didn't work due to other misaligned case_id tags, do not
+ # overwrite our otherwise ok tables!
+ if len(df_stats) == len(df):
+ df_stats.to_hdf(fdf, 'table', mode='w')
+ df_stats.to_csv(fdf.replace('.h5', '.csv'))
if __name__ == '__main__':
@@ -582,16 +602,21 @@ if __name__ == '__main__':
help='Merge all individual statistics and log file '
'analysis .csv files into one table/pd.DataFrame. '
'Requires that htc files have been created with '
- '--prep --postpro_node.')
+ '--prep --postpro_node. Combine with --zipchunks when '
+ '--prep --zipchunks was used in for generating and '
+ 'running all simulations.')
parser.add_argument('--gendlcs', default=False, action='store_true',
help='Generate DLC exchange files based on master DLC '
'spreadsheet.')
parser.add_argument('--dlcmaster', type=str, default='htc/DLCs.xlsx',
action='store', dest='dlcmaster',
- help='Master spreadsheet file location')
+ help='Optionally define an other location of the '
+ 'Master spreadsheet file location, default value is: '
+ 'htc/DLCs.xlsx')
parser.add_argument('--dlcfolder', type=str, default='htc/DLCs/',
- action='store', dest='dlcfolder', help='Destination '
- 'folder location of the generated DLC exchange files')
+ action='store', dest='dlcfolder', help='Optionally '
+ 'define an other destination folder location for the '
+ 'generated DLC exchange files, default: htc/DLCs/')
opt = parser.parse_args()
# -------------------------------------------------------------------------
@@ -633,7 +658,8 @@ if __name__ == '__main__':
print('Start creating all the htc files and pbs_in files...')
launch_dlcs_excel(sim_id, silent=False, zipchunks=opt.zipchunks,
pbs_turb=opt.pbs_turb, walltime=opt.walltime,
- postpro_node=opt.postpro_node, runmethod=RUNMETHOD)
+ postpro_node=opt.postpro_node, runmethod=RUNMETHOD,
+ dlcs_dir=os.path.join(P_SOURCE, 'htc', 'DLCs'))
# post processing: check log files, calculate statistics
if opt.check_logs or opt.stats or opt.fatigue or opt.envelopeblade \
or opt.envelopeturbine or opt.AEP:
@@ -646,7 +672,7 @@ if __name__ == '__main__':
envelopeturbine=opt.envelopeturbine,
envelopeblade=opt.envelopeblade)
if opt.postpro_node_merge:
- postpro_node_merge()
+ postpro_node_merge(zipchunks=opt.zipchunks)
if opt.dlcplot:
plot_chans = {}
plot_chans['$B1_{flap}$'] = ['setbeta-bladenr-1-flapnr-1']
diff --git a/wetb/prepost/hawcstab2.py b/wetb/prepost/hawcstab2.py
index 9eaaf88a6108a6c13210557c345b7bbff1f793af..65ac0b368b7c24d1b449edc332ea980fb3493c3b 100644
--- a/wetb/prepost/hawcstab2.py
+++ b/wetb/prepost/hawcstab2.py
@@ -64,6 +64,7 @@ def ReadFileHAWCStab2Header(fname):
return df, units
+# FIXME: with gradients currently ind has columns width of 28 instead of 14!!
class InductionResults(object):
def __init__(self):
pass
diff --git a/wetb/prepost/misc.py b/wetb/prepost/misc.py
index c6ce5759e02e3ad45abb38a248cc7d2443a2b3ea..b6129907304a3cb7c58fef4cf63f0368315312eb 100644
--- a/wetb/prepost/misc.py
+++ b/wetb/prepost/misc.py
@@ -31,7 +31,10 @@ import scipy as sp
from scipy import optimize as opt
from scipy import stats
from scipy.interpolate import griddata as interp
-from matplotlib import pyplot as plt
+try:
+ from matplotlib import pyplot as plt
+except:
+ pass
import pandas as pd