From d85254925e4fd59db68dfff238063e07d7bf0b84 Mon Sep 17 00:00:00 2001
From: mmpe <mmpe@dtu.dk>
Date: Mon, 10 Dec 2018 11:42:19 +0100
Subject: [PATCH] optional hawc2_path in HAWC2PBSFile,
---
wetb/hawc2/hawc2_pbs_file.py | 27 +++++++++++++++++----------
wetb/hawc2/htc_file_set.py | 10 +++++-----
wetb/utils/cluster_tools/pbsfile.py | 1 +
wetb/utils/tests/test_pbs_file.py | 2 ++
4 files changed, 25 insertions(+), 15 deletions(-)
diff --git a/wetb/hawc2/hawc2_pbs_file.py b/wetb/hawc2/hawc2_pbs_file.py
index 8ab247c0..53afcc3b 100644
--- a/wetb/hawc2/hawc2_pbs_file.py
+++ b/wetb/hawc2/hawc2_pbs_file.py
@@ -5,15 +5,7 @@ from wetb.utils.cluster_tools.os_path import pjoin, relpath, abspath,\
template = Template("""
-#===============================================================================
-echo copy hawc2 to scratch
-#===============================================================================
-(flock -x 200
-unzip -u -o -q [hawc2_path]/*.zip -d /scratch/$USER/$PBS_JOBID/hawc2/
-find [hawc2_path]/* ! -name *.zip -exec cp -u -t /scratch/$USER/$PBS_JOBID/hawc2/ {} +
-) 200>/scratch/$USER/$PBS_JOBID/lock_file_hawc2
-mkdir -p /scratch/$USER/$PBS_JOBID/[model_name]/run_[jobname]/[rel_exe_dir]
-cp /scratch/$USER/$PBS_JOBID/hawc2/* /scratch/$USER/$PBS_JOBID/[model_name]/run_[jobname]/[rel_exe_dir]
+[copy_hawc2]
#===============================================================================
echo copy input
@@ -91,7 +83,7 @@ class HAWC2PBSFile(PBSFile):
def commands(self):
rel_exe_dir = relpath(self.exe_dir, self.model_path)
copy_input_to_scratch, copy_input_to_exe_dir = self.copy_input()
- return template(hawc2_path=os.path.dirname(cluster_path(self.hawc2_path)),
+ return template(copy_hawc2=self.copy_hawc2(),
exe_dir=cluster_path(self.exe_dir),
copy_input_to_scratch=copy_input_to_scratch,
copy_input_to_exe_dir=copy_input_to_exe_dir,
@@ -103,6 +95,21 @@ class HAWC2PBSFile(PBSFile):
model_path=cluster_path(self.model_path),
model_name=self.model_name)
+ def copy_hawc2(self):
+ copy_hawc2 = Template("""#===============================================================================
+echo copy hawc2 to scratch
+#===============================================================================
+(flock -x 200
+unzip -u -o -q [hawc2_path]/*.zip -d /scratch/$USER/$PBS_JOBID/hawc2/
+find [hawc2_path]/* ! -name *.zip -exec cp -u -t /scratch/$USER/$PBS_JOBID/hawc2/ {} +
+) 200>/scratch/$USER/$PBS_JOBID/lock_file_hawc2
+mkdir -p /scratch/$USER/$PBS_JOBID/[model_name]/run_[jobname]/[rel_exe_dir]
+cp /scratch/$USER/$PBS_JOBID/hawc2/* /scratch/$USER/$PBS_JOBID/[model_name]/run_[jobname]/[rel_exe_dir]""")
+ if self.hawc2_path is None:
+ return ""
+ else:
+ return copy_hawc2(hawc2_path=os.path.dirname(cluster_path(self.hawc2_path)))
+
def copy_input(self):
rel_input_files = [relpath(f, self.model_path) for f in self.input_files]
diff --git a/wetb/hawc2/htc_file_set.py b/wetb/hawc2/htc_file_set.py
index 63724df0..990f52bb 100644
--- a/wetb/hawc2/htc_file_set.py
+++ b/wetb/hawc2/htc_file_set.py
@@ -23,17 +23,17 @@ class HTCFileSet():
for filename in glob.iglob(htc_path, recursive=True):
self.htc_files.append(filename)
- def pbs_files(self, hawc2_path, hawc2_cmd, htc_lst="**/*.htc", queue='workq',
+ def pbs_files(self, hawc2_path, hawc2_cmd, queue='workq', walltime=None,
input_files=None, output_files=None, copy_turb=(True, True)):
- return [HTCFile(htc).pbs_file(hawc2_path, hawc2_cmd, queue=queue,
+ return (HTCFile(htc).pbs_file(hawc2_path, hawc2_cmd, queue=queue, walltime=walltime,
input_files=copy.copy(input_files),
output_files=copy.copy(output_files),
- copy_turb=copy_turb) for htc in self.htc_files]
+ copy_turb=copy_turb) for htc in self.htc_files)
- def save_pbs_files(self, hawc2_path, hawc2_cmd, htc_lst="**/*.htc", queue='workq',
+ def save_pbs_files(self, hawc2_path=None, hawc2_cmd=JESS_WINE32_HAWC2MB, queue='workq', walltime=None,
input_files=None, output_files=None, copy_turb=(True, True)):
- for pbs in self.pbs_files(hawc2_path, hawc2_cmd, htc_lst, queue=queue,
+ for pbs in self.pbs_files(hawc2_path, hawc2_cmd, queue=queue, walltime=walltime,
input_files=input_files, output_files=output_files,
copy_turb=copy_turb):
pbs.save(self.model_path)
diff --git a/wetb/utils/cluster_tools/pbsfile.py b/wetb/utils/cluster_tools/pbsfile.py
index 7f1a64db..f4a61529 100644
--- a/wetb/utils/cluster_tools/pbsfile.py
+++ b/wetb/utils/cluster_tools/pbsfile.py
@@ -33,6 +33,7 @@ pbs_template = Template('''### Jobid
#PBS -q [queue]
cd [workdir]
mkdir -p stdout
+if [ -z "$PBS_JOBID" ]; then echo "Run using qsub"; exit ; fi
pwd
[commands]
exit
diff --git a/wetb/utils/tests/test_pbs_file.py b/wetb/utils/tests/test_pbs_file.py
index 8bd4df85..116104bb 100644
--- a/wetb/utils/tests/test_pbs_file.py
+++ b/wetb/utils/tests/test_pbs_file.py
@@ -33,6 +33,7 @@ def test_pbs_file_str():
#PBS -q workq
cd /home/user/tmp
mkdir -p stdout
+if [ -z "$PBS_JOBID" ]; then echo "Run using qsub"; exit ; fi
pwd
python -c "print('hello world')"
exit
@@ -83,6 +84,7 @@ def test_pbs_multirunner():
#PBS -q workq
cd /home/user/tmp
mkdir -p stdout
+if [ -z "$PBS_JOBID" ]; then echo "Run using qsub"; exit ; fi
pwd
echo "import os
import glob
--
GitLab