diff --git a/wetb/prepost/Simulations.py b/wetb/prepost/Simulations.py index 4081a76e60ccbde79045beabc3169664b3613e31..3fcfd4c95fef1bc7f7dba27d1432268176d0219a 100755 --- a/wetb/prepost/Simulations.py +++ b/wetb/prepost/Simulations.py @@ -2340,11 +2340,12 @@ class PBS(object): self.wine_appendix) self.pbs += ' echo "execute HAWC2, do not fork and wait"\n' self.pbs += " " + ("%s %s ./%s %s" % param).strip() + "\n" - self.pbs += ' echo "POST-PROCESSING"\n' - self.pbs += " " - self.checklogs() - self.pbs += " " - self.postprocessing() + if self.pyenv is not None: + self.pbs += ' echo "POST-PROCESSING"\n' + self.pbs += " " + self.checklogs() + self.pbs += " " + self.postprocessing() self.pbs += "fi\n" # mark end of find+xargs mode self.pbs += '# ' + '-'*78 + '\n' diff --git a/wetb/prepost/simchunks.py b/wetb/prepost/simchunks.py index 18c79bd7986188b8877f613d1c3dfbe7f8af8056..8f8c93ddaa045c0799e39af5687d6cf5aa750382 100644 --- a/wetb/prepost/simchunks.py +++ b/wetb/prepost/simchunks.py @@ -193,18 +193,19 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0, # ===================================================================== # activate the python environment - pbs += 'echo "activate python environment %s"\n' % pyenv - pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv - # sometimes activating an environment fails due to a FileExistsError - # is this because it is activated at the same time on another node? - # check twice if the environment got activated for real - pbs += 'echo "CHECK 2x IF %s IS ACTIVE, IF NOT TRY AGAIN"\n' % pyenv - pbs += 'CMD=\"from distutils.sysconfig import get_python_lib;' - pbs += 'print (get_python_lib().find(\'%s\'))"\n' % pyenv - pbs += 'ACTIVATED=`python -c "$CMD"`\n' - pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv - pbs += 'ACTIVATED=`python -c "$CMD"`\n' - pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv + if pyenv is not None: + pbs += 'echo "activate python environment %s"\n' % pyenv + pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv + # sometimes activating an environment fails due to a FileExistsError + # is this because it is activated at the same time on another node? + # check twice if the environment got activated for real + pbs += 'echo "CHECK 2x IF %s IS ACTIVE, IF NOT TRY AGAIN"\n' % pyenv + pbs += 'CMD=\"from distutils.sysconfig import get_python_lib;' + pbs += 'print (get_python_lib().find(\'%s\'))"\n' % pyenv + pbs += 'ACTIVATED=`python -c "$CMD"`\n' + pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv + pbs += 'ACTIVATED=`python -c "$CMD"`\n' + pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv # ===================================================================== # create all necessary directories at CPU_NR dirs, turb db dirs, sim_id @@ -370,7 +371,8 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0, pbs += ' --exclude *.htc \n' # when using -u, htc and pbs_in files should be ignored # pbs += 'time cp -ru %s $PBS_O_WORKDIR/\n' % tmp - pbs += 'source deactivate\n' + if pyenv is not None: + pbs += 'source deactivate\n' pbs += 'echo "DONE !!"\n' pbs += '\necho "%s"\n' % ('-'*70) pbs += '# in case wine has crashed, kill any remaining wine servers\n' diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p index fc26cf63b9534ec14d1a424cce1ebde3fbd6a40d..32dcf94fae32661e1a2fc830890d5b55070cedb2 100644 --- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p +++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p @@ -69,9 +69,6 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then else echo "execute HAWC2, do not fork and wait" time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp10_s100.htc - echo "POST-PROCESSING" - python -c "from wetb.prepost import statsdel; statsdel.logcheck('logfiles/dlc01_demos/dlc01_steady_wsp10_s100.log')" - python -c "from wetb.prepost import statsdel; statsdel.calc('res/dlc01_demos/dlc01_steady_wsp10_s100', no_bins=46, m=[3,4,6,8,9,10,12], neq=20.0, i0=0, i1=None, ftype='.csv')" fi # ------------------------------------------------------------------------------ diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p index d464f905ac1232f363356dc828f4a436781f36a9..6acf66b02eabede837a1e94581e5d96b77f8e8f3 100644 --- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p +++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p @@ -69,9 +69,6 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then else echo "execute HAWC2, do not fork and wait" time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp11_s101.htc - echo "POST-PROCESSING" - python -c "from wetb.prepost import statsdel; statsdel.logcheck('logfiles/dlc01_demos/dlc01_steady_wsp11_s101.log')" - python -c "from wetb.prepost import statsdel; statsdel.calc('res/dlc01_demos/dlc01_steady_wsp11_s101', no_bins=46, m=[3,4,6,8,9,10,12], neq=20.0, i0=0, i1=None, ftype='.csv')" fi # ------------------------------------------------------------------------------ diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p index 3ad903f889cb7e54c7ee89cfc162d1311a02d6a3..fb09a5d7109f63fe1876cf8bbd8ceb642d727d85 100644 --- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p +++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p @@ -69,9 +69,6 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then else echo "execute HAWC2, do not fork and wait" time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc - echo "POST-PROCESSING" - python -c "from wetb.prepost import statsdel; statsdel.logcheck('logfiles/dlc01_demos/dlc01_steady_wsp8_noturb.log')" - python -c "from wetb.prepost import statsdel; statsdel.calc('res/dlc01_demos/dlc01_steady_wsp8_noturb', no_bins=46, m=[3,4,6,8,9,10,12], neq=20.0, i0=0, i1=None, ftype='.csv')" fi # ------------------------------------------------------------------------------ diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p index 7afc37a7f7b65a544ab78b97208257546fb32759..ba7d6250af28e8d6bb471157c5858d31871ad188 100644 --- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p +++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p @@ -69,9 +69,6 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then else echo "execute HAWC2, do not fork and wait" time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc - echo "POST-PROCESSING" - python -c "from wetb.prepost import statsdel; statsdel.logcheck('logfiles/dlc01_demos/dlc01_steady_wsp9_noturb.log')" - python -c "from wetb.prepost import statsdel; statsdel.calc('res/dlc01_demos/dlc01_steady_wsp9_noturb', no_bins=46, m=[3,4,6,8,9,10,12], neq=20.0, i0=0, i1=None, ftype='.csv')" fi # ------------------------------------------------------------------------------ diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p new file mode 100644 index 0000000000000000000000000000000000000000..67ea5e87b40b2393ffd250e6dcb534e681c8102c --- /dev/null +++ b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p @@ -0,0 +1,158 @@ + +### Standard Output +#PBS -N remote_chnk_00000 +#PBS -o ./pbs_out_chunks/remote_chnk_00000.out +### Standard Error +#PBS -e ./pbs_out_chunks/remote_chnk_00000.err +#PBS -W umask=0003 +### Maximum wallclock time format HOURS:MINUTES:SECONDS +#PBS -l walltime=20:00:00 +#PBS -l nodes=1:ppn=12 +### Queue name +#PBS -q workq + + +echo "----------------------------------------------------------------------" +echo "activate python environment wetb_py3" +source /home/python/miniconda3/bin/activate wetb_py3 +echo "CHECK 2x IF wetb_py3 IS ACTIVE, IF NOT TRY AGAIN" +CMD="from distutils.sysconfig import get_python_lib;print (get_python_lib().find('wetb_py3'))" +ACTIVATED=`python -c "$CMD"` +if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi +ACTIVATED=`python -c "$CMD"` +if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd + +echo "create CPU directories on the scratch disk" +mkdir -p /scratch/$USER/$PBS_JOBID/remote/ +mkdir -p /scratch/$USER/$PBS_JOBID/0/ +mkdir -p /scratch/$USER/$PBS_JOBID/1/ +mkdir -p /scratch/$USER/$PBS_JOBID/2/ +mkdir -p /scratch/$USER/$PBS_JOBID/3/ +mkdir -p /scratch/$USER/$PBS_JOBID/4/ +mkdir -p /scratch/$USER/$PBS_JOBID/5/ +mkdir -p /scratch/$USER/$PBS_JOBID/6/ +mkdir -p /scratch/$USER/$PBS_JOBID/7/ +mkdir -p /scratch/$USER/$PBS_JOBID/8/ +mkdir -p /scratch/$USER/$PBS_JOBID/9/ +mkdir -p /scratch/$USER/$PBS_JOBID/10/ +mkdir -p /scratch/$USER/$PBS_JOBID/11/ + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/remote/ +echo 'current working directory:' +pwd +echo "create turb_db directories" +mkdir -p ../turb/ + +echo "----------------------------------------------------------------------" +cd $PBS_O_WORKDIR +echo 'current working directory:' +pwd +echo "get the zip-chunk file from the PBS_O_WORKDIR" +cp ./zip-chunks-gorm/remote_chnk_00000.zip /scratch/$USER/$PBS_JOBID/ + +# copy to scratch db directory for [turb_db_dir], [turb_base_name] +cp ../turb/none* /scratch/$USER/$PBS_JOBID/remote/../turb/. +cp ../turb/turb_s100_10ms* /scratch/$USER/$PBS_JOBID/remote/../turb/. +cp ../turb/turb_s101_11ms* /scratch/$USER/$PBS_JOBID/remote/../turb/. + +# copy to scratch db directory for [meand_db_dir], [meand_base_name] + +# copy to scratch db directory for [wake_db_dir], [wake_base_name] + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "unzip chunk, create dirs in cpu and sim_id folders" +/usr/bin/unzip remote_chnk_00000.zip -d 0/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 1/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 2/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 3/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 4/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 5/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 6/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 7/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 8/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 9/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 10/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 11/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d remote/. >> /dev/null + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "copy all turb files into CPU dirs" +find turb -iname *.bin -exec cp {} 0/{} \; +find turb -iname *.bin -exec cp {} 1/{} \; +find turb -iname *.bin -exec cp {} 2/{} \; +find turb -iname *.bin -exec cp {} 3/{} \; +find turb -iname *.bin -exec cp {} 4/{} \; +find turb -iname *.bin -exec cp {} 5/{} \; +find turb -iname *.bin -exec cp {} 6/{} \; +find turb -iname *.bin -exec cp {} 7/{} \; +find turb -iname *.bin -exec cp {} 8/{} \; +find turb -iname *.bin -exec cp {} 9/{} \; +find turb -iname *.bin -exec cp {} 10/{} \; +find turb -iname *.bin -exec cp {} 11/{} \; + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "START RUNNING JOBS IN find+xargs MODE" +WINEARCH=win32 WINEPREFIX=~/.wine32 winefix +# run all the PBS *.p files in find+xargs mode +echo "following cases will be run from following path:" +echo "remote/pbs_in/dlc01_demos/" +export LAUNCH_PBS_MODE=false +/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' | sort -z + +echo "number of files to be launched: "`find remote/pbs_in/dlc01_demos/ -type f | wc -l` +/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' -print0 | sort -z | /home/MET/sysalt/bin/xargs -0 -I{} --process-slot-var=CPU_NR -n 1 -P 12 sh {} +echo "END OF JOBS IN find+xargs MODE" + + +echo "----------------------------------------------------------------------" +echo 'total scratch disk usage:' +du -hs /scratch/$USER/$PBS_JOBID/ +cd /scratch/$USER/$PBS_JOBID/remote +echo 'current working directory:' +pwd +echo "Results saved at sim_id directory:" +find + +echo "move statsdel into compressed archive" +find res/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/statsdel_chnk_00000.tar +xz -z2 -T 12 prepost/statsdel_chnk_00000.tar + +echo "move log analysis into compressed archive" +find logfiles/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/loganalysis_chnk_00000.tar +xz -z2 -T 12 prepost/loganalysis_chnk_00000.tar + + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "move results back from node scratch/sim_id to origin, but ignore htc, and pbs_in directories." +echo "copy from remote/* to $PBS_O_WORKDIR/" +time rsync -au --remove-source-files remote/* $PBS_O_WORKDIR/ \ + --exclude pbs_in/dlc01_demos/* \ + --exclude *.htc +source deactivate +echo "DONE !!" + +echo "----------------------------------------------------------------------" +# in case wine has crashed, kill any remaining wine servers +# caution: ALL the users wineservers will die on this node! +echo "following wineservers are still running:" +ps -u $USER -U $USER | grep wineserver +killall -u $USER wineserver +exit diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.zip b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.zip new file mode 100644 index 0000000000000000000000000000000000000000..ffd3dc856cfbf7499c9cbeda46aff8d71a6f1067 Binary files /dev/null and b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.zip differ diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p new file mode 100644 index 0000000000000000000000000000000000000000..5c9b2f0159991ff7ccffc7a47e11f75aba1dfee1 --- /dev/null +++ b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.p @@ -0,0 +1,182 @@ + +### Standard Output +#PBS -N remote_chnk_00000 +#PBS -o ./pbs_out_chunks/remote_chnk_00000.out +### Standard Error +#PBS -e ./pbs_out_chunks/remote_chnk_00000.err +#PBS -W umask=0003 +### Maximum wallclock time format HOURS:MINUTES:SECONDS +#PBS -l walltime=20:00:00 +#PBS -l nodes=1:ppn=20 +### Queue name +#PBS -q workq + + +echo "----------------------------------------------------------------------" +echo "activate python environment wetb_py3" +source /home/python/miniconda3/bin/activate wetb_py3 +echo "CHECK 2x IF wetb_py3 IS ACTIVE, IF NOT TRY AGAIN" +CMD="from distutils.sysconfig import get_python_lib;print (get_python_lib().find('wetb_py3'))" +ACTIVATED=`python -c "$CMD"` +if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi +ACTIVATED=`python -c "$CMD"` +if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd + +echo "create CPU directories on the scratch disk" +mkdir -p /scratch/$USER/$PBS_JOBID/remote/ +mkdir -p /scratch/$USER/$PBS_JOBID/0/ +mkdir -p /scratch/$USER/$PBS_JOBID/1/ +mkdir -p /scratch/$USER/$PBS_JOBID/2/ +mkdir -p /scratch/$USER/$PBS_JOBID/3/ +mkdir -p /scratch/$USER/$PBS_JOBID/4/ +mkdir -p /scratch/$USER/$PBS_JOBID/5/ +mkdir -p /scratch/$USER/$PBS_JOBID/6/ +mkdir -p /scratch/$USER/$PBS_JOBID/7/ +mkdir -p /scratch/$USER/$PBS_JOBID/8/ +mkdir -p /scratch/$USER/$PBS_JOBID/9/ +mkdir -p /scratch/$USER/$PBS_JOBID/10/ +mkdir -p /scratch/$USER/$PBS_JOBID/11/ +mkdir -p /scratch/$USER/$PBS_JOBID/12/ +mkdir -p /scratch/$USER/$PBS_JOBID/13/ +mkdir -p /scratch/$USER/$PBS_JOBID/14/ +mkdir -p /scratch/$USER/$PBS_JOBID/15/ +mkdir -p /scratch/$USER/$PBS_JOBID/16/ +mkdir -p /scratch/$USER/$PBS_JOBID/17/ +mkdir -p /scratch/$USER/$PBS_JOBID/18/ +mkdir -p /scratch/$USER/$PBS_JOBID/19/ + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/remote/ +echo 'current working directory:' +pwd +echo "create turb_db directories" +mkdir -p ../turb/ + +echo "----------------------------------------------------------------------" +cd $PBS_O_WORKDIR +echo 'current working directory:' +pwd +echo "get the zip-chunk file from the PBS_O_WORKDIR" +cp ./zip-chunks-jess/remote_chnk_00000.zip /scratch/$USER/$PBS_JOBID/ + +# copy to scratch db directory for [turb_db_dir], [turb_base_name] +cp ../turb/none* /scratch/$USER/$PBS_JOBID/remote/../turb/. +cp ../turb/turb_s100_10ms* /scratch/$USER/$PBS_JOBID/remote/../turb/. +cp ../turb/turb_s101_11ms* /scratch/$USER/$PBS_JOBID/remote/../turb/. + +# copy to scratch db directory for [meand_db_dir], [meand_base_name] + +# copy to scratch db directory for [wake_db_dir], [wake_base_name] + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "unzip chunk, create dirs in cpu and sim_id folders" +/usr/bin/unzip remote_chnk_00000.zip -d 0/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 1/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 2/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 3/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 4/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 5/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 6/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 7/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 8/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 9/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 10/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 11/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 12/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 13/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 14/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 15/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 16/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 17/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 18/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d 19/. >> /dev/null +/usr/bin/unzip remote_chnk_00000.zip -d remote/. >> /dev/null + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "copy all turb files into CPU dirs" +find turb -iname *.bin -exec cp {} 0/{} \; +find turb -iname *.bin -exec cp {} 1/{} \; +find turb -iname *.bin -exec cp {} 2/{} \; +find turb -iname *.bin -exec cp {} 3/{} \; +find turb -iname *.bin -exec cp {} 4/{} \; +find turb -iname *.bin -exec cp {} 5/{} \; +find turb -iname *.bin -exec cp {} 6/{} \; +find turb -iname *.bin -exec cp {} 7/{} \; +find turb -iname *.bin -exec cp {} 8/{} \; +find turb -iname *.bin -exec cp {} 9/{} \; +find turb -iname *.bin -exec cp {} 10/{} \; +find turb -iname *.bin -exec cp {} 11/{} \; +find turb -iname *.bin -exec cp {} 12/{} \; +find turb -iname *.bin -exec cp {} 13/{} \; +find turb -iname *.bin -exec cp {} 14/{} \; +find turb -iname *.bin -exec cp {} 15/{} \; +find turb -iname *.bin -exec cp {} 16/{} \; +find turb -iname *.bin -exec cp {} 17/{} \; +find turb -iname *.bin -exec cp {} 18/{} \; +find turb -iname *.bin -exec cp {} 19/{} \; + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "START RUNNING JOBS IN find+xargs MODE" +WINEARCH=win32 WINEPREFIX=~/.wine32 winefix +# run all the PBS *.p files in find+xargs mode +echo "following cases will be run from following path:" +echo "remote/pbs_in/dlc01_demos/" +export LAUNCH_PBS_MODE=false +/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' | sort -z + +echo "number of files to be launched: "`find remote/pbs_in/dlc01_demos/ -type f | wc -l` +/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' -print0 | sort -z | /home/MET/sysalt/bin/xargs -0 -I{} --process-slot-var=CPU_NR -n 1 -P 20 sh {} +echo "END OF JOBS IN find+xargs MODE" + + +echo "----------------------------------------------------------------------" +echo 'total scratch disk usage:' +du -hs /scratch/$USER/$PBS_JOBID/ +cd /scratch/$USER/$PBS_JOBID/remote +echo 'current working directory:' +pwd +echo "Results saved at sim_id directory:" +find + +echo "move statsdel into compressed archive" +find res/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/statsdel_chnk_00000.tar +xz -z2 -T 20 prepost/statsdel_chnk_00000.tar + +echo "move log analysis into compressed archive" +find logfiles/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/loganalysis_chnk_00000.tar +xz -z2 -T 20 prepost/loganalysis_chnk_00000.tar + + +echo "----------------------------------------------------------------------" +cd /scratch/$USER/$PBS_JOBID/ +echo 'current working directory:' +pwd +echo "move results back from node scratch/sim_id to origin, but ignore htc, and pbs_in directories." +echo "copy from remote/* to $PBS_O_WORKDIR/" +time rsync -au --remove-source-files remote/* $PBS_O_WORKDIR/ \ + --exclude pbs_in/dlc01_demos/* \ + --exclude *.htc +source deactivate +echo "DONE !!" + +echo "----------------------------------------------------------------------" +# in case wine has crashed, kill any remaining wine servers +# caution: ALL the users wineservers will die on this node! +echo "following wineservers are still running:" +ps -u $USER -U $USER | grep wineserver +killall -u $USER wineserver +exit diff --git a/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.zip b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.zip new file mode 100644 index 0000000000000000000000000000000000000000..ffd3dc856cfbf7499c9cbeda46aff8d71a6f1067 Binary files /dev/null and b/wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-jess/remote_chnk_00000.zip differ diff --git a/wetb/prepost/tests/test_Simulations.py b/wetb/prepost/tests/test_Simulations.py index f4da4ea23c6525a0c67d265a92e495b2fd63cd54..e765d9d7b66f6ff3bfb8dc0367d39b9010513963 100644 --- a/wetb/prepost/tests/test_Simulations.py +++ b/wetb/prepost/tests/test_Simulations.py @@ -14,13 +14,14 @@ import unittest import os import filecmp import shutil +from zipfile import ZipFile import numpy as np import pandas as pd from wetb.prepost import dlctemplate as tmpl from wetb.prepost import Simulations as sim -from wetb.fatigue_tools.fatigue import eq_load +from wetb.prepost.misc import DictDiff class Template(unittest.TestCase): @@ -28,6 +29,14 @@ class Template(unittest.TestCase): self.basepath = os.path.dirname(__file__) +class TestErrorLogs(Template): + + def test_loganalysis(self): + # select a few log cases and do the analysis, save to csv and convert + # saved result to DataFrame + pass + + class TestGenerateInputs(Template): def test_launch_dlcs_excel(self): @@ -55,19 +64,38 @@ class TestGenerateInputs(Template): tmpl.force_dir = tmpl.P_RUN tmpl.launch_dlcs_excel('remote', silent=True, runmethod='gorm', - pbs_turb=True) + pbs_turb=True, zipchunks=True) # we can not check-in empty dirs so we can not compare the complete # directory structure withouth manually creating the empty dirs here for subdir in ['control', 'data', 'htc', 'pbs_in', 'pbs_in_turb', - 'htc/_master', 'htc/dlc01_demos', 'pbs_in/dlc01_demos']: + 'htc/_master', 'htc/dlc01_demos', 'pbs_in/dlc01_demos', + 'zip-chunks-gorm', 'zip-chunks-jess']: remote = os.path.join(p_root, tmpl.PROJECT, 'remote', subdir) ref = os.path.join(p_root, tmpl.PROJECT, 'ref', subdir) - cmp = filecmp.dircmp(remote, ref) + # the zipfiles are taken care of separately + ignore = ['remote_chnk_00000.zip'] + cmp = filecmp.dircmp(remote, ref, ignore=ignore) self.assertEqual(len(cmp.diff_files), 0, cmp.diff_files) self.assertEqual(len(cmp.right_only), 0, cmp.right_only) self.assertEqual(len(cmp.left_only), 0, cmp.left_only) + # compare the zip files + for fname in ['demo_dlc_remote.zip', + 'zip-chunks-gorm/remote_chnk_00000.zip', + 'zip-chunks-jess/remote_chnk_00000.zip']: + remote = os.path.join(p_root, tmpl.PROJECT, 'remote', fname) + ref = os.path.join(p_root, tmpl.PROJECT, 'ref', fname) + + with ZipFile(remote) as zrem, ZipFile(ref) as zref: + self.assertEqual(len(zrem.infolist()), len(zref.infolist())) + frem = {f.filename:f.file_size for f in zrem.infolist()} + fref = {f.filename:f.file_size for f in zref.infolist()} + dd = DictDiff(frem, fref) + self.assertEqual(len(dd.added()), 0) + self.assertEqual(len(dd.removed()), 0) + self.assertEqual(len(dd.changed()), 0) + # for the pickled file we can just read it remote = os.path.join(p_root, tmpl.PROJECT, 'remote', 'prepost') ref = os.path.join(p_root, tmpl.PROJECT, 'ref', 'prepost')