Skip to content
Snippets Groups Projects
Commit 7c28969f authored by David Verelst's avatar David Verelst
Browse files

prepost.dlctemplate/simchunks: remove redundant keyword, explicitly

refer to python environment on cluster
parent f0fd071a
No related branches found
No related tags found
No related merge requests found
...@@ -338,11 +338,11 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False, ...@@ -338,11 +338,11 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
# respective nodes. It is not walltime per case. # respective nodes. It is not walltime per case.
sorts_on = ['[DLC]', '[Windspeed]'] sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=20, create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=20,
nr_procs_series=9, processes=1, nr_procs_series=9, walltime='20:00:00',
walltime='20:00:00', chunks_dir='zip-chunks-jess') chunks_dir='zip-chunks-jess')
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=12, create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=12,
nr_procs_series=15, processes=1, nr_procs_series=15, walltime='20:00:00',
walltime='20:00:00', chunks_dir='zip-chunks-gorm') chunks_dir='zip-chunks-gorm')
df = sim.Cases(cases).cases2df() df = sim.Cases(cases).cases2df()
df.to_excel(os.path.join(POST_DIR, sim_id + '.xls')) df.to_excel(os.path.join(POST_DIR, sim_id + '.xls'))
......
...@@ -32,10 +32,9 @@ import pandas as pd ...@@ -32,10 +32,9 @@ import pandas as pd
from wetb.prepost.Simulations import Cases from wetb.prepost.Simulations import Cases
def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
nr_procs_series=9, processes=1, queue='workq', nr_procs_series=9, queue='workq', pyenv='wetb_py3',
walltime='24:00:00', chunks_dir='zip-chunks-jess', walltime='24:00:00', chunks_dir='zip-chunks-jess'):
pyenv='wetb_py3', i0=0):
"""Group a large number of simulations htc and pbs launch scripts into """Group a large number of simulations htc and pbs launch scripts into
different zip files so we can run them with find+xargs on various nodes. different zip files so we can run them with find+xargs on various nodes.
""" """
...@@ -187,7 +186,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, ...@@ -187,7 +186,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20,
# ===================================================================== # =====================================================================
# activate the python environment # activate the python environment
pbs += 'echo "activate python environment %s"\n' % pyenv pbs += 'echo "activate python environment %s"\n' % pyenv
pbs += 'source activate %s\n' % pyenv pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv
# sometimes activating an environment fails due to a FileExistsError # sometimes activating an environment fails due to a FileExistsError
# is this because it is activated at the same time on another node? # is this because it is activated at the same time on another node?
# check twice if the environment got activated for real # check twice if the environment got activated for real
...@@ -396,6 +395,10 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, ...@@ -396,6 +395,10 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20,
except (FileExistsError, OSError): except (FileExistsError, OSError):
pass pass
fpath = os.path.join(df['[run_dir]'].iloc[0], 'pbs_out_chunks')
if not os.path.exists(fpath):
os.makedirs(fpath)
df_iter = chunker(df, nr_procs_series*ppn) df_iter = chunker(df, nr_procs_series*ppn)
sim_id = df['[sim_id]'].iloc[0] sim_id = df['[sim_id]'].iloc[0]
run_dir = df['[run_dir]'].iloc[0] run_dir = df['[run_dir]'].iloc[0]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment