From 7c28969f385cb0a0628fa58d207f684da0611caa Mon Sep 17 00:00:00 2001
From: David Robert Verelst <dave@dtu.dk>
Date: Mon, 10 Apr 2017 17:50:45 +0200
Subject: [PATCH] prepost.dlctemplate/simchunks: remove redundant keyword,
explicitly refer to python environment on cluster
---
wetb/prepost/dlctemplate.py | 8 ++++----
wetb/prepost/simchunks.py | 13 ++++++++-----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
index e056518..beb9d54 100644
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -338,11 +338,11 @@ def launch_dlcs_excel(sim_id, silent=False, verbose=False, pbs_turb=False,
# respective nodes. It is not walltime per case.
sorts_on = ['[DLC]', '[Windspeed]']
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=20,
- nr_procs_series=9, processes=1,
- walltime='20:00:00', chunks_dir='zip-chunks-jess')
+ nr_procs_series=9, walltime='20:00:00',
+ chunks_dir='zip-chunks-jess')
create_chunks_htc_pbs(cases, sort_by_values=sorts_on, ppn=12,
- nr_procs_series=15, processes=1,
- walltime='20:00:00', chunks_dir='zip-chunks-gorm')
+ nr_procs_series=15, walltime='20:00:00',
+ chunks_dir='zip-chunks-gorm')
df = sim.Cases(cases).cases2df()
df.to_excel(os.path.join(POST_DIR, sim_id + '.xls'))
diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py
index 4e4d560..0fc2f8e 100644
--- a/wetb/prepost/simchunks.py
+++ b/wetb/prepost/simchunks.py
@@ -32,10 +32,9 @@ import pandas as pd
from wetb.prepost.Simulations import Cases
-def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20,
- nr_procs_series=9, processes=1, queue='workq',
- walltime='24:00:00', chunks_dir='zip-chunks-jess',
- pyenv='wetb_py3', i0=0):
+def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
+ nr_procs_series=9, queue='workq', pyenv='wetb_py3',
+ walltime='24:00:00', chunks_dir='zip-chunks-jess'):
"""Group a large number of simulations htc and pbs launch scripts into
different zip files so we can run them with find+xargs on various nodes.
"""
@@ -187,7 +186,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20,
# =====================================================================
# activate the python environment
pbs += 'echo "activate python environment %s"\n' % pyenv
- pbs += 'source activate %s\n' % pyenv
+ pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv
# sometimes activating an environment fails due to a FileExistsError
# is this because it is activated at the same time on another node?
# check twice if the environment got activated for real
@@ -396,6 +395,10 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20,
except (FileExistsError, OSError):
pass
+ fpath = os.path.join(df['[run_dir]'].iloc[0], 'pbs_out_chunks')
+ if not os.path.exists(fpath):
+ os.makedirs(fpath)
+
df_iter = chunker(df, nr_procs_series*ppn)
sim_id = df['[sim_id]'].iloc[0]
run_dir = df['[run_dir]'].iloc[0]
--
GitLab