diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py index e056518739425b28646324a6a502f2b7f12f8d87..beb9d5424b28d5ff539b26b4d3ab1c6d202f0ef4 100644 --- a/wetb/prepost/dlctemplate.py +++ b/wetb/prepost/dlctemplate.py @@ -338,11 +338,11 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False, # respective nodes. It is not walltime per case. sorts_on = ['[DLC]', '[Windspeed]'] create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=20, - nr_procs_series=9, processes=1, - walltime='20:00:00', chunks_dir='zip-chunks-jess') + nr_procs_series=9, walltime='20:00:00', + chunks_dir='zip-chunks-jess') create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=12, - nr_procs_series=15, processes=1, - walltime='20:00:00', chunks_dir='zip-chunks-gorm') + nr_procs_series=15, walltime='20:00:00', + chunks_dir='zip-chunks-gorm') df = sim.Cases(cases).cases2df() df.to_excel(os.path.join(POST_DIR, sim_id + '.xls')) diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py index 4e4d560313bf88756054d24e0dd5981a4bca617b..0fc2f8eff7501e3ccc26cb14de71518a6c2df8f1 100644 --- a/wetb/prepost/simchunks.py +++ b/wetb/prepost/simchunks.py @@ -32,10 +32,9 @@ import pandas as pd from wetb.prepost.Simulations import Cases -def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, - nr_procs_series=9, processes=1, queue='workq', - walltime='24:00:00', chunks_dir='zip-chunks-jess', - pyenv='wetb_py3', i0=0): +def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0, + nr_procs_series=9, queue='workq', pyenv='wetb_py3', + walltime='24:00:00', chunks_dir='zip-chunks-jess'): """Group a large number of simulations htc and pbs launch scripts into different zip files so we can run them with find+xargs on various nodes. """ @@ -187,7 +186,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, # ===================================================================== # activate the python environment pbs += 'echo "activate python environment %s"\n' % pyenv - pbs += 'source activate %s\n' % pyenv + pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv # sometimes activating an environment fails due to a FileExistsError # is this because it is activated at the same time on another node? # check twice if the environment got activated for real @@ -396,6 +395,10 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, except (FileExistsError, OSError): pass + fpath = os.path.join(df['[run_dir]'].iloc[0], 'pbs_out_chunks') + if not os.path.exists(fpath): + os.makedirs(fpath) + df_iter = chunker(df, nr_procs_series*ppn) sim_id = df['[sim_id]'].iloc[0] run_dir = df['[run_dir]'].iloc[0]