diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
index 2572d17d8469ba4ae30b08d0e9cd7ea2cb40b69e..e622d7ff63cbd7300436b577f66c94cc752e0764 100644
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -266,7 +266,7 @@ def variable_tag_func_mod1(master, case_id_short=False):
def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
runmethod=None, write_htc=True, zipchunks=False,
walltime='04:00:00', postpro_node=False,
- dlcs_dir='htc/DLCs'):
+ dlcs_dir='htc/DLCs', compress=False):
"""
Launch load cases defined in Excel files
"""
@@ -346,10 +346,10 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=20,
nr_procs_series=3, walltime='20:00:00',
- chunks_dir='zip-chunks-jess')
+ chunks_dir='zip-chunks-jess', compress=compress)
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=12,
nr_procs_series=3, walltime='20:00:00',
- chunks_dir='zip-chunks-gorm')
+ chunks_dir='zip-chunks-gorm', compress=compress)
df = sim.Cases(cases).cases2df()
df.to_excel(os.path.join(POST_DIR, sim_id + '.xls'))
@@ -617,6 +617,9 @@ if __name__ == '__main__':
parser.add_argument('--zipchunks', default=False, action='store_true',
dest='zipchunks', help='Create PBS launch files for'
'running in zip-chunk find+xargs mode.')
+ parser.add_argument('--compress', default=False, action='store_true',
+ dest='compress', help='When running in zip-chunk mode,'
+ 'compress log and results files into chunks.')
parser.add_argument('--pbs_turb', default=False, action='store_true',
dest='pbs_turb', help='Create PBS launch files to '
'create the turbulence boxes in stand alone mode '
@@ -694,7 +697,8 @@ if __name__ == '__main__':
launch_dlcs_excel(sim_id, silent=False, zipchunks=opt.zipchunks,
pbs_turb=opt.pbs_turb, walltime=opt.walltime,
postpro_node=opt.postpro_node, runmethod=RUNMETHOD,
- dlcs_dir=os.path.join(P_SOURCE, 'htc', 'DLCs'))
+ dlcs_dir=os.path.join(P_SOURCE, 'htc', 'DLCs'),
+ compress=opt.compress)
# post processing: check log files, calculate statistics
if opt.check_logs or opt.stats or opt.fatigue or opt.envelopeblade \
or opt.envelopeturbine or opt.AEP:
diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py
index 7c224d4a7d8c91ba4c983d2e105212a25257259e..b06f923c234eb5bfb8745fff78058cdca155b736 100644
--- a/wetb/prepost/simchunks.py
+++ b/wetb/prepost/simchunks.py
@@ -36,7 +36,8 @@ from wetb.prepost.Simulations import Cases
def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
nr_procs_series=9, queue='workq', pyenv='wetb_py3',
- walltime='24:00:00', chunks_dir='zip-chunks-jess'):
+ walltime='24:00:00', chunks_dir='zip-chunks-jess',
+ compress=False):
"""Group a large number of simulations htc and pbs launch scripts into
different zip files so we can run them with find+xargs on various nodes.
"""
@@ -146,7 +147,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
"""
- def make_pbs_chunks(df, ii, sim_id, run_dir, model_zip):
+ def make_pbs_chunks(df, ii, sim_id, run_dir, model_zip, compress=False):
"""Create a PBS that:
* copies all required files (zip chunk) to scratch disk
* copies all required turbulence files to scratch disk
@@ -315,25 +316,25 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
rpl = (os.path.join(pbs_in_base, '*'), os.path.join(htc_base, '*'))
pbs += 'find \n'
- # compress all result files into an archive, first *.sel files
- # FIXME: why doesn this work with -name "*.sel" -o -name "*.dat"??
- pbs += '\necho "move results into compressed archive"\n'
- pbs += 'find %s -name "*.sel" -print0 ' % res_base
- fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
- pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
- # now add the *.dat files to the archive
- pbs += 'find %s -name "*.dat" -print0 ' % res_base
- fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
- pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
-
- pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
-
- # compress all logfiles into an archive
- pbs += '\necho "move logfiles into compressed archive"\n'
- pbs += 'find %s -name "*.log" -print0 ' % log_base
- fname = os.path.join(log_base, 'logfiles_chnk_%05i' % ii)
- pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
- pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
+ if compress:
+ # compress all result files into an archive, first *.sel files
+ # FIXME: why doesn this work with -name "*.sel" -o -name "*.dat"??
+ pbs += '\necho "move results into compressed archive"\n'
+ pbs += 'find %s -name "*.sel" -print0 ' % res_base
+ fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
+ pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
+ # now add the *.dat files to the archive
+ pbs += 'find %s -name "*.dat" -print0 ' % res_base
+ fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
+ pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
+ pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
+
+ # compress all logfiles into an archive
+ pbs += '\necho "move logfiles into compressed archive"\n'
+ pbs += 'find %s -name "*.log" -print0 ' % log_base
+ fname = os.path.join(log_base, 'logfiles_chnk_%05i' % ii)
+ pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
+ pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
# compress all post-processing results (saved as csv's) into an archive
pbs += '\necho "move statsdel into compressed archive"\n'
@@ -411,7 +412,8 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
df_ind.index.name = '[case_id]'
for ii, dfi in enumerate(df_iter):
fname, ind = make_zip_chunks(dfi, i0+ii, sim_id, run_dir, model_zip)
- make_pbs_chunks(dfi, i0+ii, sim_id, run_dir, model_zip)
+ make_pbs_chunks(dfi, i0+ii, sim_id, run_dir, model_zip,
+ compress=compress)
df_ind = df_ind.append(ind)
print(fname)