From 5be816743cc64a288b242ddaeb0621e929129258 Mon Sep 17 00:00:00 2001
From: David Robert Verelst <dave@dtu.dk>
Date: Thu, 11 Oct 2018 11:09:17 +0200
Subject: [PATCH] prepost.simchunks: use correct scratch disc path for each
 cpu/turb dir

---
 wetb/prepost/simchunks.py                     |  7 ++-
 .../ref/zip-chunks-gorm/remote_chnk_00000.p   | 28 ++++++------
 .../ref/zip-chunks-jess/remote_chnk_00000.p   | 44 +++++++++----------
 3 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py
index 899e629..45acab2 100644
--- a/wetb/prepost/simchunks.py
+++ b/wetb/prepost/simchunks.py
@@ -361,8 +361,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
                     continue
                 db_dir_abs = os.path.join(pbase, sim_id, db_dir, '')
                 for k in list(range(ppn)):
-                    turb_dir_abs = os.path.join(pbase, sim_id, str(k),
-                                                turb_dir, '')
+                    turb_dir_abs = os.path.join(pbase, str(k), turb_dir, '')
                     rpl = (db_dir_abs, turb_dir_abs)
                     pbs += 'find %s -iname "*.bin" -exec ln -s {} %s \\;\n' % rpl
 
@@ -412,7 +411,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
         pbs += 'pwd\n'
         pbs += 'echo "Results saved at sim_id directory:"\n'
         rpl = (os.path.join(pbs_in_base, '*'), os.path.join(htc_base, '*'))
-        pbs += 'find \n'
+        pbs += 'find .\n'
 
         if compress:
             # compress all result files into an archive, first *.sel files
@@ -460,7 +459,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
         pbs += 'echo "copy from %s to $PBS_O_WORKDIR/"\n' % tmp
         pbs += 'time rsync -au --remove-source-files %s $PBS_O_WORKDIR/ \\\n' % tmp
         pbs += '    --exclude %s \\\n' % os.path.join(pbs_in_base, '*')
-        pbs += '    --exclude *.htc \n'
+        pbs += '    --exclude *.htc\n'
         # when using -u, htc and pbs_in files should be ignored
 #        pbs += 'time cp -ru %s $PBS_O_WORKDIR/\n' % tmp
         if pyenv is not None:
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p
index 4f403f2..7683139 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p
@@ -112,18 +112,18 @@ cd /scratch/$USER/$PBS_JOBID/remote/
 echo 'current working directory:'
 pwd
 echo "Link all turb files into CPU dirs"
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/0/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/1/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/2/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/3/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/4/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/5/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/6/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/7/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/8/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/9/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/10/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/11/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/0/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/1/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/2/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/3/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/4/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/5/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/6/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/7/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/8/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/9/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/10/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/11/turb/ \;
 
 echo "----------------------------------------------------------------------"
 cd /scratch/$USER/$PBS_JOBID/
@@ -149,7 +149,7 @@ cd /scratch/$USER/$PBS_JOBID/remote
 echo 'current working directory:'
 pwd
 echo "Results saved at sim_id directory:"
-find 
+find .
 
 echo "move statsdel into compressed archive"
 find res/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/statsdel_chnk_00000.tar
@@ -168,7 +168,7 @@ echo "move results back from node scratch/sim_id to origin, but ignore htc, and
 echo "copy from remote/* to $PBS_O_WORKDIR/"
 time rsync -au --remove-source-files remote/* $PBS_O_WORKDIR/ \
     --exclude pbs_in/dlc01_demos/* \
-    --exclude *.htc 
+    --exclude *.htc
 source deactivate
 echo "DONE !!"
 
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p
index baf98fa..330c907 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p
@@ -136,26 +136,26 @@ cd /scratch/$USER/$PBS_JOBID/remote/
 echo 'current working directory:'
 pwd
 echo "Link all turb files into CPU dirs"
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/0/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/1/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/2/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/3/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/4/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/5/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/6/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/7/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/8/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/9/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/10/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/11/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/12/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/13/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/14/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/15/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/16/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/17/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/18/turb/ \;
-find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/19/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/0/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/1/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/2/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/3/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/4/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/5/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/6/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/7/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/8/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/9/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/10/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/11/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/12/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/13/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/14/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/15/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/16/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/17/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/18/turb/ \;
+find /scratch/$USER/$PBS_JOBID/remote/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/19/turb/ \;
 
 echo "----------------------------------------------------------------------"
 cd /scratch/$USER/$PBS_JOBID/
@@ -181,7 +181,7 @@ cd /scratch/$USER/$PBS_JOBID/remote
 echo 'current working directory:'
 pwd
 echo "Results saved at sim_id directory:"
-find 
+find .
 
 echo "move statsdel into compressed archive"
 find res/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/statsdel_chnk_00000.tar
@@ -200,7 +200,7 @@ echo "move results back from node scratch/sim_id to origin, but ignore htc, and
 echo "copy from remote/* to $PBS_O_WORKDIR/"
 time rsync -au --remove-source-files remote/* $PBS_O_WORKDIR/ \
     --exclude pbs_in/dlc01_demos/* \
-    --exclude *.htc 
+    --exclude *.htc
 source deactivate
 echo "DONE !!"
 
-- 
GitLab