diff --git a/wetb/prepost/dlcdefs.py b/wetb/prepost/dlcdefs.py
index 1f4e754df00522503865343de486cba8a045966f..a287d2a3832a8f23b89f2f72a257273497c4cb27 100644
--- a/wetb/prepost/dlcdefs.py
+++ b/wetb/prepost/dlcdefs.py
@@ -391,7 +391,7 @@ def excel_stabcon(proot, fext='xlsx', pignore=None, pinclude=None, sheet=0,
if not silent:
k = 0
- for df in dict_dfs:
+ for dlc, df in viewitems(dict_dfs):
k += len(df)
print('in which a total of %i cases are defined.' % k)
diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
index 8308b17518f9276224435ce5ecb0681fa86a87f0..cd0b651c58e1074b2abb119937783d8f5a4dc7b2 100644
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -374,7 +374,7 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
ppn=20, nr_procs_series=3, walltime='20:00:00',
chunks_dir='zip-chunks-jess', compress=compress,
wine_arch=wine_arch, wine_prefix=wine_prefix,
- prelude=prelude)
+ prelude=prelude, ppn_pbs=20)
# create_chunks_htc_pbs(cases, sort_by_values=sorts_on, queue='workq',
# ppn=12, nr_procs_series=3, walltime='20:00:00',
# chunks_dir='zip-chunks-gorm', compress=compress,
@@ -598,8 +598,8 @@ def postpro_node_merge(tqdm=False, zipchunks=False, m=[3,4,6,8,9,10,12]):
cc = sim.Cases(POST_DIR, sim_id)
df_tags = cc.cases2df()
df_stats = pd.merge(df, df_tags[required], on=['[case_id]'])
- # if the merge didn't work due to other misaligned case_id tags, do not
- # overwrite our otherwise ok tables!
+ # find out if we have some misalignment between generated cases and results
+ # this could happen when we added new cases and removed others
if len(df_stats) != len(df):
print('failed to merge required tags, something is wrong!')
# find out which cases we lost and why
@@ -610,8 +610,12 @@ def postpro_node_merge(tqdm=False, zipchunks=False, m=[3,4,6,8,9,10,12]):
msg = 'nr of case_ids lost:'
print(msg, (len(df)-len(df_stats))/len(df['channel'].unique()))
print('following case_ids have mysteriously disappeared:')
- print(s_df-s_stats)
- return
+ missing = s_df-s_stats
+ print(missing)
+ # save misalligned cases
+ fname = os.path.join(POST_DIR, '%s_misallgined_cases.tsv' % sim_id)
+ pd.DataFrame(missing).to_csv(fname, sep='\t')
+
df_stats.to_hdf(fdf, 'table', mode='w')
df_stats.to_csv(fdf.replace('.h5', '.csv'))
@@ -626,7 +630,7 @@ def postpro_node_merge(tqdm=False, zipchunks=False, m=[3,4,6,8,9,10,12]):
def prepare_failed(compress=False, wine_arch='win32', wine_prefix='~/.wine32',
prelude='', zipchunks=False):
- cc = sim.Cases(POST_DIR, sim_id)
+ cc = sim.Cases(POST_DIR, sim_id, rem_failed=False)
df_tags = cc.cases2df()
# -------------------------------------------------------------------------
diff --git a/wetb/prepost/misc.py b/wetb/prepost/misc.py
index b6129907304a3cb7c58fef4cf63f0368315312eb..caa424d00df9cd892e570c50ce0fbe72f3fd1c33 100644
--- a/wetb/prepost/misc.py
+++ b/wetb/prepost/misc.py
@@ -1113,8 +1113,10 @@ def df_dict_check_datatypes(df_dict):
# we can not pop/delete items from a dict while iterating over it
df_dict2 = {}
for colkey, col in df_dict.items():
+ if len(col)==0:
+ pass
# if we have a list, convert to string
- if type(col[0]).__name__ == 'list':
+ elif type(col[0]).__name__ == 'list':
for ii, item in enumerate(col):
col[ii] = '**'.join(item)
# if we already have an array (statistics) or a list of numbers
diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py
index a6498c63ca8c6ab71da880e92729c4b14507fad3..f514aebbfa22c19000c4ea588e755fc3c3a90b57 100644
--- a/wetb/prepost/simchunks.py
+++ b/wetb/prepost/simchunks.py
@@ -39,7 +39,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
walltime='24:00:00', chunks_dir='zip-chunks-jess',
wine_arch='win32', wine_prefix='~/.wine32',
pyenv_cmd='source /home/python/miniconda3/bin/activate',
- pyenv='wetb_py3', prelude=''):
+ pyenv='wetb_py3', prelude='', ppn_pbs=20):
"""Group a large number of simulations htc and pbs launch scripts into
different zip files so we can run them with find+xargs on various nodes.
"""
@@ -145,7 +145,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
pbs_tmplate += "#PBS -W umask=[umask]\n"
pbs_tmplate += "### Maximum wallclock time format HOURS:MINUTES:SECONDS\n"
pbs_tmplate += "#PBS -l walltime=[walltime]\n"
- pbs_tmplate += "#PBS -l nodes=[nodes]:ppn=[ppn]\n"
+ pbs_tmplate += "#PBS -l nodes=[nodes]:ppn=[ppn_pbs]\n"
pbs_tmplate += "### Queue name\n"
pbs_tmplate += "#PBS -q [queue]\n"
pbs_tmplate += "\n"
@@ -210,7 +210,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
pbs = pbs.replace('[umask]', '0003')
pbs = pbs.replace('[walltime]', walltime)
pbs = pbs.replace('[nodes]', str(nodes))
- pbs = pbs.replace('[ppn]', str(ppn))
+ pbs = pbs.replace('[ppn_pbs]', str(ppn_pbs))
pbs = pbs.replace('[queue]', queue)
pbs += '\necho "%s"\n' % ('-'*70)