Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • toolbox/WindEnergyToolbox
  • tlbl/WindEnergyToolbox
  • cpav/WindEnergyToolbox
  • frza/WindEnergyToolbox
  • borg/WindEnergyToolbox
  • mmpe/WindEnergyToolbox
  • ozgo/WindEnergyToolbox
  • dave/WindEnergyToolbox
  • mmir/WindEnergyToolbox
  • wluo/WindEnergyToolbox
  • welad/WindEnergyToolbox
  • chpav/WindEnergyToolbox
  • rink/WindEnergyToolbox
  • shfe/WindEnergyToolbox
  • shfe1/WindEnergyToolbox
  • acdi/WindEnergyToolbox
  • angl/WindEnergyToolbox
  • wliang/WindEnergyToolbox
  • mimc/WindEnergyToolbox
  • wtlib/WindEnergyToolbox
  • cmos/WindEnergyToolbox
  • fabpi/WindEnergyToolbox
22 results
Show changes
Hide whitespace changes
Inline Side-by-side
Showing
with 755 additions and 583 deletions
wetb/prepost/prepost.py
View file @ 62f0f2e4
......@@ -4,14 +4,6 @@ Created on Tue Mar 10 18:47:32 2015
@author: dave
"""
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from io import open
from future import standard_library
standard_library.install_aliases()
import os
import copy
......
wetb/prepost/simchunks.py
View file @ 62f0f2e4
......@@ -4,22 +4,9 @@ Created on Mon Aug 8 10:22:49 2016
@author: dave
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
from builtins import dict
from io import open
from builtins import zip
from builtins import range
from builtins import str
from builtins import int
from future import standard_library
standard_library.install_aliases()
from builtins import object
# standard python library
import os
from os.path import join as pjoin
import zipfile
import copy
import tarfile
......@@ -32,12 +19,15 @@ import pandas as pd
#from tqdm import tqdm
from wetb.prepost.Simulations import Cases
from wetb.prepost import misc
def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
nr_procs_series=9, queue='workq', pyenv='wetb_py3',
nr_procs_series=9, queue='workq', compress=False,
walltime='24:00:00', chunks_dir='zip-chunks-jess',
compress=False, wine_64bit=False):
wine_arch='win32', wine_prefix='.wine32',
pyenv_cmd='source /home/python/miniconda3/bin/activate',
pyenv='py36-wetb', prelude='', ppn_pbs=20):
"""Group a large number of simulations htc and pbs launch scripts into
different zip files so we can run them with find+xargs on various nodes.
"""
......@@ -66,11 +56,12 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
fname_model = os.path.join(run_dir, model_zip)
with zipfile.ZipFile(fname_model, 'r') as zf_model:
for n in zf_model.namelist():
# FIXME: might be duplicates when creating dirs from dirtags
zf.writestr(n, zf_model.open(n).read())
# create all necessary directories in the zip file
dirtags = ['[htc_dir]', '[res_dir]','[log_dir]','[animation_dir]',
'[pbs_in_dir]', '[eigenfreq_dir]','[turb_dir]','[wake_dir]',
'[pbs_in_dir]', '[eigenfreq_dir]','[turb_dir]','[micro_dir]',
'[meander_dir]','[hydro_dir]', '[mooring_dir]',
'[pbs_in_dir]', '[pbs_out_dir]']
dirnames = []
......@@ -78,13 +69,16 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
for dirname in set(df[tag].unique().tolist()):
if not dirname or dirname.lower() not in ['false', 'none', 0]:
dirnames.append(dirname)
namelist = set(zf.namelist())
for dirname in set(dirnames):
if dirname != 0:
zf.write('.', os.path.join(dirname, '.'))
if dirname != 0 and pjoin(dirname, '') not in namelist:
# FIXME: might have duplicates from the base model zip file
zf.write('.', arcname=os.path.join(dirname, '.'))
# and the post-processing data
# FIXME: do not use hard coded paths!
zf.write('.', 'prepost-data/')
zf.write('.', arcname='prepost-data/')
# HTC files
df_src = df['[run_dir]'] + df['[htc_dir]'] + df['[case_id]']
......@@ -97,7 +91,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
# Since df_src and df_dst are already Series, iterating is fast an it
# is slower to first convert to a list
for src, dst_rel in zip(df_src, df_dst):
zf.write(src+'.htc', dst_rel+'.htc')
zf.write(src+'.htc', arcname=dst_rel+'.htc')
# PBS files
df_src = df['[run_dir]'] + df['[pbs_in_dir]'] + df['[case_id]']
......@@ -105,7 +99,7 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
# Since df_src and df_dst are already Series, iterating is fast an it
# is slower to first convert to a list
for src, dst_rel in zip(df_src, df_dst):
zf.write(src+'.p', dst_rel+'.p')
zf.write(src+'.p', arcname=dst_rel+'.p')
# copy and rename input files with given versioned name to the
# all files that will have to be renamed to their non-changeable
......@@ -115,40 +109,63 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
copyto_files = []
# cycle through the unique elements
for k in set(copyto_files_tmp):
# k is of form: "['some/file.txt', 'another/file1.txt']"
if len(k) < 2:
# k is of form: 'some/file.txt**another/file1.txt
if len(k) < 1:
continue
items = [kk[1:-1] for kk in k.split('[')[1].split(']')[0].split(', ')]
copyto_files.extend(items)
# note that *;* is done in df_dict_check_datatypes()
copyto_files.extend(k.split('*;*'))
# we might still have non unique elements
copyto_files = set(copyto_files)
for copyto_file, dst_rel in zip(copyto_files, df_dst):
src = os.path.join(run_dir, copyto_file)
# make dir if it does not exist
zf.write('.', os.path.dirname(copyto_file), '.')
zf.write(src, copyto_file)
namelist = set(zf.namelist())
# write an empty directory if applicable, make sure ends with /
copyto_file_folder = pjoin(os.path.dirname(copyto_file), '')
if len(copyto_file_folder) > 0:
if copyto_file_folder not in namelist:
zf.write('.', arcname=copyto_file_folder)
# if we have a wildcard, copy all files accordingly
for fname in glob.glob(copyto_file, recursive=True):
print(fname)
if copyto_file not in namelist:
zf.write(fname, arcname=fname)
zf.close()
return fname, df_index
pbs_tmplate ="""
### Standard Output
#PBS -N [job_name]
#PBS -o [std_out]
### Standard Error
#PBS -e [std_err]
#PBS -W umask=[umask]
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=[walltime]
#PBS -l nodes=[nodes]:ppn=[ppn]
### Queue name
#PBS -q [queue]
"""
pbs_tmplate = "\n"
pbs_tmplate += "### Standard Output\n"
pbs_tmplate += "#PBS -N [job_name]\n"
pbs_tmplate += "#PBS -o [std_out]\n"
pbs_tmplate += "### Standard Error\n"
pbs_tmplate += "#PBS -e [std_err]\n"
pbs_tmplate += "#PBS -W umask=[umask]\n"
pbs_tmplate += "### Maximum wallclock time format HOURS:MINUTES:SECONDS\n"
pbs_tmplate += "#PBS -l walltime=[walltime]\n"
pbs_tmplate += "#PBS -l nodes=[nodes]:ppn=[ppn_pbs]\n"
pbs_tmplate += "### Queue name\n"
pbs_tmplate += "#PBS -q [queue]\n"
pbs_tmplate += "\n"
# FIXME: this causes troubles on CI runner for the tests (line endings?)
# pbs_tmplate = """
# ### Standard Output
# #PBS -N [job_name]
# #PBS -o [std_out]
# ### Standard Error
# #PBS -e [std_err]
# #PBS -W umask=[umask]
# ### Maximum wallclock time format HOURS:MINUTES:SECONDS
# #PBS -l walltime=[walltime]
# #PBS -l nodes=[nodes]:ppn=[ppn]
# ### Queue name
# #PBS -q [queue]
# """
def make_pbs_chunks(df, ii, sim_id, run_dir, model_zip, compress=False,
wine_64bit=False):
wine_arch='win32', wine_prefix='.wine32'):
"""Create a PBS that:
* copies all required files (zip chunk) to scratch disk
* copies all required turbulence files to scratch disk
......@@ -160,9 +177,10 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
cmd_xargs = '/home/MET/sysalt/bin/xargs'
jobid = '%s_chnk_%05i' % (sim_id, ii)
wineparam = ('win32', '~/.wine32')
if wine_64bit:
wineparam = ('win64', '~/.wine')
# sanitize wine_prefix
wine_prefix = misc.sanitize_wine_prefix(wine_prefix)
wineparam = (wine_arch, wine_prefix)
pbase = os.path.join('/scratch','$USER', '$PBS_JOBID', '')
post_dir_base = post_dir.split(sim_id)[1]
......@@ -192,38 +210,45 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
pbs = pbs.replace('[std_err]', './pbs_out_chunks/%s.err' % jobid)
pbs = pbs.replace('[umask]', '0003')
pbs = pbs.replace('[walltime]', walltime)
pbs = pbs.replace('[nodes]', str(nodes))
pbs = pbs.replace('[ppn]', str(ppn))
pbs = pbs.replace('[nodes]', str(1)) # only one node for the time being
pbs = pbs.replace('[ppn_pbs]', str(ppn_pbs))
pbs = pbs.replace('[queue]', queue)
pbs += '\necho "%s"\n' % ('-'*70)
# run prelude code
# =====================================================================
pbs += prelude
# =====================================================================
# activate the python environment
if pyenv is not None:
pbs += 'echo "activate python environment %s"\n' % pyenv
pbs += 'source /home/python/miniconda3/bin/activate %s\n' % pyenv
rpl = (pyenv_cmd, pyenv)
pbs += '%s %s\n' % rpl
# sometimes activating an environment fails due to a FileExistsError
# is this because it is activated at the same time on another node?
# check twice if the environment got activated for real
pbs += 'echo "CHECK 2x IF %s IS ACTIVE, IF NOT TRY AGAIN"\n' % pyenv
pbs += 'CMD=\"from distutils.sysconfig import get_python_lib;'
pbs += 'print (get_python_lib().find(\'%s\'))"\n' % pyenv
pbs += 'ACTIVATED=`python -c "$CMD"`\n'
pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv
pbs += 'ACTIVATED=`python -c "$CMD"`\n'
pbs += 'if [ $ACTIVATED -eq -1 ]; then source activate %s;fi\n' % pyenv
# check twice if the environment got activated for real,
# but only do so for /home/python/miniconda
if pyenv_cmd.find('miniconda') > -1:
pbs += 'echo "CHECK 2x IF %s IS ACTIVE, IF NOT TRY AGAIN"\n' % pyenv
pbs += 'CMD=\"from distutils.sysconfig import get_python_lib;'
pbs += 'print (get_python_lib().find(\'/usr/lib/python\'))"\n'
pbs += 'ACTIVATED=`python -c "$CMD"`\n'
pbs += 'if [ $ACTIVATED -eq 0 ]; then %s %s;fi\n' % rpl
pbs += 'ACTIVATED=`python -c "$CMD"`\n'
pbs += 'if [ $ACTIVATED -eq 0 ]; then %s %s;fi\n' % rpl
# =====================================================================
# create all necessary directories at CPU_NR dirs
# browse to scratch directory
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % pbase
pbs += 'cd "%s"\n' % pbase
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n\n'
pbs += 'echo "create CPU directories on the scratch disk"\n'
pbs += 'mkdir -p %s\n' % os.path.join(pbase, sim_id, '')
pbs += 'mkdir -p "%s"\n' % os.path.join(pbase, sim_id, '')
for k in range(ppn):
pbs += 'mkdir -p %s\n' % os.path.join(pbase, '%i' % k, '')
pbs += 'mkdir -p "%s"\n' % os.path.join(pbase, '%i' % k, '')
# =====================================================================
# get the zip-chunk file from the PBS_O_WORKDIR
......@@ -234,13 +259,13 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
pbs += 'pwd\n'
pbs += 'echo "get the zip-chunk file from the PBS_O_WORKDIR"\n'
# copy the relevant zip chunk file to the scratch main directory
rpl = (os.path.join('./', chunks_dir, jobid), os.path.join(pbase, ''))
pbs += 'cp %s.zip %s\n' % rpl
rpl = (os.path.join(chunks_dir, jobid), os.path.join(pbase, ''))
pbs += 'cp "%s.zip" "%s"\n' % rpl
# =====================================================================
# unzip to all cpu dirs
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % pbase
pbs += 'cd "%s"\n' % pbase
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "unzip chunk, create dirs in cpu and sim_id folders"\n'
......@@ -248,16 +273,16 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
# contains files defined in [copyto_files]
for k in list(range(ppn)) + [sim_id]:
dst = os.path.join('%s' % k, '.')
pbs += '/usr/bin/unzip %s -d %s >> /dev/null\n' % (jobid+'.zip', dst)
pbs += '/usr/bin/unzip "%s" -d "%s" >> /dev/null\n' % (jobid+'.zip', dst)
# =====================================================================
# create all turb_db directories
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % os.path.join(pbase, sim_id, '')
pbs += 'cd "%s"\n' % os.path.join(pbase, sim_id, '')
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "create turb_db directories"\n'
turb_db_tags = ['[turb_db_dir]', '[meand_db_dir]', '[wake_db_dir]']
turb_db_tags = ['[turb_db_dir]', '[meander_db_dir]', '[micro_db_dir]']
turb_db_dirs = []
for tag in turb_db_tags:
for dirname in set(df[tag].unique().tolist()):
......@@ -266,56 +291,76 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
turb_db_dirs.append(dirname)
turb_db_dirs = set(turb_db_dirs)
# create all turb dirs
for dirname in turb_db_dirs:
pbs += 'mkdir -p %s\n' % os.path.join(dirname, '')
for dirname in sorted(turb_db_dirs):
pbs += 'mkdir -p "%s"\n' % os.path.join(dirname, '')
# =====================================================================
# copy required turbulence from db_dir to scratch/db_dirs
# turb_db_dir might not be set, same for turb_base_name, for those
# cases we do not need to copy anything from the database to the node
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd $PBS_O_WORKDIR\n'
pbs += 'cd "$PBS_O_WORKDIR"\n'
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
base_name_tags = ['[turb_base_name]', '[meand_base_name]',
'[wake_base_name]']
base_name_tags = ['[turb_base_name]', '[meander_base_name]',
'[micro_base_name]']
for db, base_name in zip(turb_db_tags, base_name_tags):
turb_db_dirs = df[db] + df[base_name]
# When set to None, the DataFrame will have text as None
# FIXME: CI runner has and old pandas version
# FIXME: CI runner has and old pandas version (v0.14.1)
try:
turb_db_src = turb_db_dirs[turb_db_dirs.str.find('None')==-1]
p1 = turb_db_dirs.str.find('None')==-1
p2 = turb_db_dirs.str.find('none')==-1
p3 = turb_db_dirs.str.find('false')==-1
p4 = turb_db_dirs.str.find('False')==-1
# p4 = turb_db_dirs.str.find('0')==-1
turb_db_src = turb_db_dirs[p1 & p2 & p3 & p4]
except AttributeError:
turb_db_src = turb_db_dirs[turb_db_dirs.str.findall('None')==-1]
# and findall returns list with the search str occuring as
# many times as found in the str...
# sel should be True if str does NOT occur in turb_db_dirs
# meaning if findall returns empty list
sel = [True]*len(turb_db_dirs)
for val in ['false', 'none', 'None', 'False']:
findall = turb_db_dirs.str.findall(val).tolist()
# len==0 if nothing has been found
sel_ = [True if len(k)==0 else False for k in findall]
# merge with other search results, none of the elements
# should occur
sel = [True if k and kk else False for k, kk in zip(sel, sel_)]
turb_db_src = turb_db_dirs[sel]
pbs += '\n'
pbs += '# copy to scratch db directory for %s, %s\n' % (db, base_name)
for k in turb_db_src.unique():
for k in sorted(turb_db_src.unique()):
dst = os.path.dirname(os.path.join(pbase, sim_id, k))
pbs += 'cp %s* %s\n' % (k, os.path.join(dst, '.'))
# globbing doesn't work in either single- or double-quotes.
# However, you can interpolate globbing with double-quoted strings
# https://unix.stackexchange.com/a/67761/163108
pbs += 'cp "%s"*.bin "%s"\n' % (k, os.path.join(dst, '.'))
# =====================================================================
# to be safe, create all turb dirs in the cpu dirs
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % os.path.join(pbase, sim_id, '')
pbs += 'cd "%s"\n' % os.path.join(pbase, '')
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "create turb directories in CPU dirs"\n'
turb_dir_tags = ['[turb_dir]', '[meand_dir]', '[wake_dir]']
turb_dir_tags = ['[turb_dir]', '[meander_dir]', '[micro_dir]']
turb_dirs = []
for tag in turb_dir_tags:
for dirname in set(df[tag].unique().tolist()):
dirname_s = str(dirname).replace('.', '').replace('/', '')
if dirname_s.lower() not in ['false', 'none', '0']:
turb_dirs.append(dirname)
turb_dirs = set(turb_dirs)
turb_dirs = sorted(set(turb_dirs))
for k in list(range(ppn)):
for dirname in turb_dirs:
pbs += 'mkdir -p %s\n' % os.path.join(str(k), dirname, '')
pbs += 'mkdir -p "%s"\n' % os.path.join(str(k), dirname, '')
# =====================================================================
# symlink everything from the turb_db_dir to the cpu/turb_dir
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % os.path.join(pbase, sim_id, '')
pbs += 'cd "%s"\n' % os.path.join(pbase, sim_id, '')
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "Link all turb files into CPU dirs"\n'
......@@ -334,15 +379,13 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
db_dir, turb_dir = symlink.split('_*_')
db_dir_s = db_dir.replace('.', '').replace('/', '').lower()
turb_dir_s = turb_dir.replace('.', '').replace('/', '').lower()
print(db_dir_s, turb_dir_s)
if db_dir_s in nogo or turb_dir_s in nogo:
continue
db_dir_abs = os.path.join(pbase, db_dir, '')
db_dir_abs = os.path.join(pbase, sim_id, db_dir, '')
for k in list(range(ppn)):
turb_dir_abs = os.path.join(pbase, sim_id, str(k),
turb_dir, '')
turb_dir_abs = os.path.join(pbase, str(k), turb_dir, '')
rpl = (db_dir_abs, turb_dir_abs)
pbs += 'find %s -iname "*.bin" -exec ln -s {} %s \\;\n' % rpl
pbs += 'find "%s" -iname "*.bin" -exec ln -s {} "%s" \\;\n' % rpl
# copy all from scratch/turb_db to cpu/turb
# turb_dir_base = os.path.join(compath(list(turb_dirs)), '')
......@@ -359,22 +402,23 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
# finally we can run find+xargs!!!
pbs += '\n'
pbs += 'echo "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % pbase
pbs += 'cd "%s"\n' % pbase
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "START RUNNING JOBS IN find+xargs MODE"\n'
pbs += 'WINEARCH=%s WINEPREFIX=%s winefix\n' % wineparam
if wine_arch!=None or wine_prefix!=None:
pbs += 'WINEARCH="%s" WINEPREFIX="%s" winefix\n' % wineparam
pbs += '# run all the PBS *.p files in find+xargs mode\n'
pbs += 'echo "following cases will be run from following path:"\n'
pbs += 'echo "%s"\n' % (os.path.join(sim_id, pbs_in_base))
pbs += 'export LAUNCH_PBS_MODE=false\n'
rpl = (cmd_find, os.path.join(sim_id, pbs_in_base))
pbs += "%s %s -type f -name '*.p' | sort -z\n" % rpl
pbs += "%s '%s' -type f -name '*.p' | sort -z\n" % rpl
pbs += '\n'
pbs += 'echo "number of files to be launched: "'
pbs += '`find %s -type f | wc -l`\n' % os.path.join(sim_id, pbs_in_base)
pbs += '`find "%s" -type f | wc -l`\n' % os.path.join(sim_id, pbs_in_base)
rpl = (cmd_find, os.path.join(sim_id, pbs_in_base), cmd_xargs, ppn)
cmd = ("%s %s -type f -name '*.p' -print0 | sort -z | %s -0 -I{} "
cmd = ("%s '%s' -type f -name '*.p' -print0 | sort -z | %s -0 -I{} "
"--process-slot-var=CPU_NR -n 1 -P %i sh {}\n" % rpl)
pbs += cmd
pbs += 'echo "END OF JOBS IN find+xargs MODE"\n'
......@@ -383,77 +427,82 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
# move results back from the node sim_id dir to the origin
pbs += '\n'
pbs += '\necho "%s"\n' % ('-'*70)
pbs += "echo 'total scratch disk usage:'\n"
pbs += 'du -hs %s\n' % pbase
pbs += 'cd %s\n' % os.path.join(pbase, sim_id)
pbs += "echo 'current working directory:'\n"
pbs += 'echo "total scratch disk usage:"\n'
pbs += 'du -hs "%s"\n' % pbase
pbs += 'cd "%s"\n' % os.path.join(pbase, sim_id)
pbs += 'echo "current working directory:"\n'
pbs += 'pwd\n'
pbs += 'echo "Results saved at sim_id directory:"\n'
rpl = (os.path.join(pbs_in_base, '*'), os.path.join(htc_base, '*'))
pbs += 'find \n'
pbs += 'find .\n'
if compress:
# compress all result files into an archive, first *.sel files
# FIXME: why doesn this work with -name "*.sel" -o -name "*.dat"??
pbs += '\necho "move results into compressed archive"\n'
pbs += 'find %s -name "*.sel" -print0 ' % res_base
pbs += 'find "%s" -name "*.sel" -print0 ' % res_base
fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
# now add the *.dat files to the archive
pbs += 'find %s -name "*.dat" -print0 ' % res_base
pbs += 'find "%s" -name "*.dat" -print0 ' % res_base
fname = os.path.join(res_base, 'resfiles_chnk_%05i' % ii)
pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
pbs += '| xargs -0 tar --remove-files -rf "%s.tar"\n' % fname
pbs += 'xz -z2 -T %i "%s.tar"\n' % (ppn, fname)
# compress all logfiles into an archive
pbs += '\necho "move logfiles into compressed archive"\n'
pbs += 'find %s -name "*.log" -print0 ' % log_base
pbs += 'find "%s" -name "*.log" -print0 ' % log_base
fname = os.path.join(log_base, 'logfiles_chnk_%05i' % ii)
pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
pbs += '| xargs -0 tar --remove-files -rf "%s.tar"\n' % fname
pbs += 'xz -z2 -T %i "%s.tar"\n' % (ppn, fname)
# compress all post-processing results (saved as csv's) into an archive
pbs += '\necho "move statsdel into compressed archive"\n'
pbs += 'find %s -name "*.csv" -print0 ' % res_base
pbs += 'find "%s" -name "*.csv" -print0 ' % res_base
fname = os.path.join(post_dir_base, 'statsdel_chnk_%05i' % ii)
pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
pbs += '| xargs -0 tar --remove-files -rf "%s.tar"\n' % fname
pbs += 'xz -z2 -T %i "%s.tar"\n' % (ppn, fname)
# compress all post-processing results (saved as csv's) into an archive
pbs += '\necho "move log analysis into compressed archive"\n'
pbs += 'find %s -name "*.csv" -print0 ' % log_base
pbs += 'find "%s" -name "*.csv" -print0 ' % log_base
fname = os.path.join(post_dir_base, 'loganalysis_chnk_%05i' % ii)
pbs += '| xargs -0 tar --remove-files -rf %s.tar\n' % fname
pbs += 'xz -z2 -T %i %s.tar\n' % (ppn, fname)
pbs += '| xargs -0 tar --remove-files -rf "%s.tar"\n' % fname
pbs += 'xz -z2 -T %i "%s.tar"\n' % (ppn, fname)
pbs += '\n'
pbs += '\necho "%s"\n' % ('-'*70)
pbs += 'cd %s\n' % pbase
pbs += 'cd "%s"\n' % pbase
pbs += "echo 'current working directory:'\n"
pbs += 'pwd\n'
pbs += 'echo "move results back from node scratch/sim_id to origin, '
pbs += 'but ignore htc, and pbs_in directories."\n'
tmp = os.path.join(sim_id, '*')
tmp = os.path.join(sim_id, '') # make we have trailing /
pbs += 'echo "copy from %s to $PBS_O_WORKDIR/"\n' % tmp
pbs += 'time rsync -au --remove-source-files %s $PBS_O_WORKDIR/ \\\n' % tmp
pbs += ' --exclude %s \\\n' % os.path.join(pbs_in_base, '*')
pbs += ' --exclude *.htc \n'
pbs += 'time rsync -au "%s" "$PBS_O_WORKDIR/" \\\n' % tmp
pbs += ' --exclude "%s" \\\n' % os.path.join(pbs_in_base, '*')
pbs += ' --exclude *.htc\n'
# when using -u, htc and pbs_in files should be ignored
# pbs += 'time cp -ru %s $PBS_O_WORKDIR/\n' % tmp
if pyenv is not None:
pbs += 'source deactivate\n'
pbs += 'echo "DONE !!"\n'
pbs += '\necho "%s"\n' % ('-'*70)
pbs += '# in case wine has crashed, kill any remaining wine servers\n'
pbs += '# caution: ALL the users wineservers will die on this node!\n'
pbs += 'echo "following wineservers are still running:"\n'
pbs += 'ps -u $USER -U $USER | grep wineserver\n'
pbs += 'killall -u $USER wineserver\n'
if wine_arch!=None or wine_prefix!=None:
pbs += '# in case wine has crashed, kill any remaining wine servers\n'
pbs += '# caution: ALL the users wineservers will die on this node!\n'
pbs += 'echo "following wineservers are still running:"\n'
pbs += 'ps -u $USER -U $USER | grep wineserver\n'
pbs += 'killall -u $USER wineserver\n'
pbs += 'exit\n'
rpl = (sim_id, ii)
fname = os.path.join(run_dir, chunks_dir, '%s_chnk_%05i' % rpl)
if pbs.find('rm -rf') > -1 or pbs.find('rm -fr') > -1:
raise UserWarning('Anything that looks like rm -rf is prohibited.')
with open(fname+'.p', 'w') as f:
f.write(pbs)
......@@ -483,25 +532,81 @@ def create_chunks_htc_pbs(cases, sort_by_values=['[Windspeed]'], ppn=20, i0=0,
if not os.path.exists(fpath):
os.makedirs(fpath)
df_iter = chunker(df, nr_procs_series*ppn)
# remove all cases that start with test_ and move them into a separate
# chunk
try:
sel_notest = df['[Case folder]'].str.find('test_') < 0
if '[hs2]' in df.columns:
# depends on if '' or ';' was translated into 0/1 or not
if df['[hs2]'].dtype == np.dtype('O'):
sel_hs2 = df['[hs2]'].str.find(';') < 0
else:
sel_hs2 = df['[hs2]'] == 1
else:
sel_hs2 = pd.Series(data=False, index=df.index)
# remove hs2 cases from notest
sel_notest = sel_notest & ~sel_hs2
# tests but not HS2
sel_test = ~sel_notest & ~sel_hs2
except AttributeError:
# FIXME: CI runner has and old pandas version (v0.14.1)
# and findall returns list with the search str occuring as
# many times as found in the str...
findall = df['[Case folder]'].str.findall('test_').tolist()
# len==0 if nothing has been found
sel_notest = [True if len(k)==0 else False for k in findall]
if '[hs2]' in df.columns:
if df['[hs2]'].dtype == np.dtype('O'):
findall = df['[hs2]'].str.findall(';').tolist()
else:
findall = (df['[hs2]']==1).tolist()
# only select it if we do NOT find ;
sel_hs2 = [True if len(k)==0 else False for k in findall]
else:
sel_hs2 = [False]*len(findall)
# remove hs2 cases from notest
sel_notest = [i and not j for (i,j) in zip(sel_notest, sel_hs2)]
# tests but not HS2, so everything that is not in either notest or hs2
sel_test = [not i and not j for (i,j) in zip(sel_notest, sel_hs2)]
df_dlc = df[sel_notest]
df_test = df[sel_test]
df_hs2 = df[sel_hs2]
# DLC CHUNKS
sim_id = df['[sim_id]'].iloc[0]
run_dir = df['[run_dir]'].iloc[0]
model_zip = df['[model_zip]'].iloc[0]
post_dir = df['[post_dir]'].iloc[0]
nodes = 1
df_ind = pd.DataFrame(columns=['chnk_nr'], dtype=np.int32)
df_ind.index.name = '[case_id]'
for ii, dfi in enumerate(df_iter):
fname, ind = make_zip_chunks(dfi, i0+ii, sim_id, run_dir, model_zip)
make_pbs_chunks(dfi, i0+ii, sim_id, run_dir, model_zip,
compress=compress, wine_64bit=wine_64bit)
df_ind = df_ind.append(ind)
print(fname)
fname = os.path.join(post_dir, 'case_id-chunk-index')
df_ind['chnk_nr'] = df_ind['chnk_nr'].astype(np.int32)
df_ind.to_hdf(fname+'.h5', 'table', compression=9, complib='zlib')
df_ind.to_csv(fname+'.csv')
names = ['case_id-chunk-index', 'case_id-chunk-test-index',
'case_id-chunk-hs2-index']
i0s = [0, 90000, 80000]
# group test_ hs2 and normal dlc's in 3 different chunks
for df, name, i02 in zip([df_dlc, df_test, df_hs2], names, i0s):
i02 = i02 + i0
df_ind = pd.DataFrame(columns=['chnk_nr'], dtype=np.int32)
df_ind.index.name = '[case_id]'
df_iter = chunker(df, nr_procs_series*ppn)
for ii, dfi in enumerate(df_iter):
fname, ind = make_zip_chunks(dfi, i02+ii, sim_id, run_dir, model_zip)
make_pbs_chunks(dfi, i02+ii, sim_id, run_dir, model_zip,
wine_arch=wine_arch, wine_prefix=wine_prefix,
compress=compress)
df_ind = pd.concat([df_ind, ind])
print(fname)
fname = os.path.join(post_dir, 'case_id-chunk-index')
df_ind['chnk_nr'] = df_ind['chnk_nr'].astype(np.int32)
df_ind.to_hdf(fname+'.h5', 'table', complevel=9, complib='zlib')
df_ind.to_csv(fname+'.csv')
def regroup_tarfiles(cc):
......@@ -606,6 +711,9 @@ def merge_from_tarfiles(df_fname, path, pattern, tarmode='r:xz', tqdm=False,
def tqdm(itereable):
return itereable
if len(glob.glob(os.path.join(path, pattern))) < 1:
raise RuntimeError('No files found to merge')
for tar_fname in tqdm(glob.glob(os.path.join(path, pattern))):
if verbose:
print(tar_fname)
......@@ -743,6 +851,12 @@ class AppendDataFrames(object):
recursive
Return
------
header_fjoined : str
String of the header that was written to the joined file.
"""
if isinstance(header, int):
write_header = True
......@@ -762,12 +876,16 @@ class AppendDataFrames(object):
if write_header:
if header_fjoined is None:
header_fjoined = lines[header]
header_f0 = lines[header]
# add extra column with the file name if applicable
if fname_col:
rpl = sep + fname_col + '\n'
header_fjoined = header_fjoined.replace('\n', rpl)
ft.write(header_fjoined)
write_header = False
# check if the header is the same for each file
if header is not None:
assert lines[header] == header_f0
# but cut out the header on all other occurances
case_id = '.'.join(case_id.split('.')[:-1])
for line in lines[icut:]:
......@@ -778,6 +896,7 @@ class AppendDataFrames(object):
# and move from temp dir to fjoined
shutil.move(ftname, fjoined)
return header_fjoined.replace('\n', '')
def csv2df_chunks(self, store, fcsv, chunksize=100000, min_itemsize={},
colnames=None, dtypes={}, header='infer', sep=';'):
......
wetb/prepost/statsdel.py
View file @ 62f0f2e4
......@@ -4,20 +4,6 @@ Created on Thu Jul 21 15:13:38 2016
@author: dave
"""
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import
#from builtins import dict
#from io import open
#from builtins import zip
#from builtins import range
#from builtins import str
#from builtins import int
from future import standard_library
standard_library.install_aliases()
import os
from wetb.prepost import windIO
......
wetb/prepost/tests/data/campbell_diagram.cmb
View file @ 62f0f2e4
# Wind [m/s] 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10
0.400000E+01 0.228933E+00 0.233973E+00 0.762776E+00 0.965817E+00 0.136002E+01 0.149639E+01 0.159423E+01 0.202127E+01 0.204610E+01 0.208646E+01 0.388822E+00 0.966751E+01 0.102903E+01 0.118873E+01 0.224277E+02 0.237007E+01 0.156716E+02 0.304645E+01 0.155964E+02 0.431867E+01
0.500000E+01 0.228944E+00 0.234004E+00 0.763301E+00 0.966955E+00 0.136149E+01 0.149576E+01 0.159339E+01 0.202086E+01 0.204910E+01 0.208435E+01 0.400712E+00 0.973227E+01 0.106285E+01 0.133466E+01 0.224476E+02 0.237088E+01 0.156581E+02 0.304886E+01 0.155855E+02 0.426180E+01
0.600000E+01 0.228950E+00 0.233713E+00 0.764151E+00 0.968410E+00 0.136286E+01 0.149520E+01 0.159243E+01 0.202039E+01 0.205218E+01 0.208226E+01 0.406516E+00 0.960985E+01 0.113674E+01 0.151399E+01 0.224486E+02 0.236761E+01 0.156309E+02 0.304953E+01 0.155767E+02 0.419995E+01
0.700000E+01 0.228969E+00 0.233943E+00 0.765058E+00 0.970449E+00 0.136386E+01 0.149485E+01 0.159148E+01 0.201989E+01 0.205516E+01 0.208023E+01 0.410362E+00 0.956280E+01 0.125579E+01 0.172544E+01 0.225723E+02 0.236204E+01 0.156830E+02 0.304991E+01 0.156414E+02 0.414330E+01
......
wetb/prepost/tests/data/controller_input_quadratic.txt
View file @ 62f0f2e4
......@@ -11,3 +11,22 @@ K1 = 7.30949 [deg], K2 = 1422.81187 [deg^2] (dq/dtheta = -176.489
Additional terms due to the Aerodynamic damping
Kp2 = 0.240394E-01 [rad/(rad/s)]
Ko1 = -1.69769 [deg], Ko2 = -15.02688 [deg^2] (dq/domega = 243.08924 kNm/(rad/s))
********************************************
Aerodynamic gains:
********************************************
(1) theta [deg] (2) dq/dtheta [kNm/deg] (3) fit [kNm/deg] (4) dq/domega [kNm/(rad/s)] (5) fit [kNm/(rad/s)]
0.00000 -1165.04860 -1182.80164 -393.03157 -950.85937
4.10000 -1665.72575 -1655.44826 -6919.03943 -6544.84749
6.69000 -2012.86015 -1998.12171 -13119.30826 -12659.67192
8.62000 -2290.61883 -2275.67536 -18911.31597 -18515.75425
10.26000 -2535.50152 -2526.42508 -24632.87239 -24364.04365
11.74000 -2757.11114 -2764.46364 -30186.31522 -30329.61030
13.10000 -2991.31463 -2993.03195 -36257.79933 -36386.82912
14.38000 -3213.58048 -3216.75546 -42410.93450 -42591.10977
15.59000 -3428.46978 -3435.91220 -48626.47812 -48904.89826
16.76000 -3642.91400 -3654.91116 -55070.40445 -55424.76312
17.88000 -3858.46084 -3871.07886 -61702.38984 -62048.05630
18.97000 -4075.53879 -4087.58722 -68581.71761 -68852.77188
20.03000 -4295.29300 -4303.93692 -75700.65394 -75809.68369
21.05000 -4524.66782 -4517.52214 -83045.36607 -82820.10608
22.05000 -4758.62680 -4732.06052 -90639.34883 -89993.97031
wetb/prepost/tests/data/demo_dlc/ref/demo_dlc_remote.zip
View file @ 62f0f2e4
No preview for this file type
wetb/prepost/tests/data/demo_dlc/ref/htc/DLCs/dlc01_demos.xlsx
View file @ 62f0f2e4
No preview for this file type
wetb/prepost/tests/data/demo_dlc/ref/htc/dlc01_demos/dlc01_steady_wsp11_s101.htc
View file @ 62f0f2e4
......@@ -382,7 +382,7 @@ begin wind ;
filename_u ./turb/turb_s101_11msu.bin ;
filename_v ./turb/turb_s101_11msv.bin ;
filename_w ./turb/turb_s101_11msw.bin ;
box_dim_u 512 0.85938 ;
box_dim_u 512 0.859375 ;
box_dim_v 32 7.5;
box_dim_w 32 7.5;
std_scaling 1.0 0.7 0.5 ;
......
wetb/prepost/tests/data/demo_dlc/ref/htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc
View file @ 62f0f2e4
......@@ -379,10 +379,10 @@ begin wind ;
;
begin mann ;
create_turb_parameters 29.4 1.0 3.9 0 1.0 ; L, alfaeps, gamma, seed, highfrq compensation
filename_u ./turb/noneu.bin ;
filename_v ./turb/nonev.bin ;
filename_w ./turb/nonew.bin ;
box_dim_u 512 0.03906 ;
filename_u ./turb/Noneu.bin ;
filename_v ./turb/Nonev.bin ;
filename_w ./turb/Nonew.bin ;
box_dim_u 512 0.0390625 ;
box_dim_v 32 7.5;
box_dim_w 32 7.5;
std_scaling 1.0 0.7 0.5 ;
......
wetb/prepost/tests/data/demo_dlc/ref/htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc
View file @ 62f0f2e4
......@@ -379,10 +379,10 @@ begin wind ;
;
begin mann ;
create_turb_parameters 29.4 1.0 3.9 0 1.0 ; L, alfaeps, gamma, seed, highfrq compensation
filename_u ./turb/noneu.bin ;
filename_v ./turb/nonev.bin ;
filename_w ./turb/nonew.bin ;
box_dim_u 512 0.04395 ;
filename_u ./turb/Noneu.bin ;
filename_v ./turb/Nonev.bin ;
filename_w ./turb/Nonew.bin ;
box_dim_u 512 0.0439453 ;
box_dim_v 32 7.5;
box_dim_w 32 7.5;
std_scaling 1.0 0.7 0.5 ;
......
wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p
View file @ 62f0f2e4
### Standard Output
#PBS -N dlc01_steady_wsp10_s100
#PBS -o ./pbs_out/dlc01_demos/dlc01_steady_wsp10_s100.out
#PBS -o 'pbs_out/dlc01_demos/dlc01_steady_wsp10_s100.out'
### Standard Error
#PBS -e ./pbs_out/dlc01_demos/dlc01_steady_wsp10_s100.err
#PBS -e 'pbs_out/dlc01_demos/dlc01_steady_wsp10_s100.err'
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=04:00:00
......@@ -15,10 +15,10 @@
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### Create scratch directory and copy data to it
cd $PBS_O_WORKDIR
echo "current working dir (pwd):"
cd "$PBS_O_WORKDIR"
echo 'current working dir (pwd):'
pwd
cp -R ./demo_dlc_remote.zip /scratch/$USER/$PBS_JOBID
cp -R 'demo_dlc_remote.zip' "/scratch/$USER/$PBS_JOBID"
fi
# ==============================================================================
......@@ -29,20 +29,20 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo
echo 'Execute commands on scratch nodes'
cd /scratch/$USER/$PBS_JOBID
cd "/scratch/$USER/$PBS_JOBID"
# create unique dir for each CPU
mkdir "1"; cd "1"
mkdir '1'; cd '1'
pwd
/usr/bin/unzip ../demo_dlc_remote.zip
mkdir -p htc/dlc01_demos/
mkdir -p res/dlc01_demos/
mkdir -p logfiles/dlc01_demos/
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp10_s100.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/turb_s100_10ms*.bin turb/
mkdir -p 'htc/dlc01_demos/'
mkdir -p 'res/dlc01_demos/'
mkdir -p 'logfiles/dlc01_demos/'
mkdir -p 'turb/'
cp -R "$PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp10_s100.htc" 'htc/dlc01_demos/'
cp -R "$PBS_O_WORKDIR/../turb/turb_s100_10ms"*.bin "turb/"
_HOSTNAME_=`hostname`
if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
WINEARCH='win32' WINEPREFIX="$HOME/.wine32" winefix
fi
# ==============================================================================
......@@ -54,21 +54,21 @@ else
fi
# ------------------------------------------------------------------------------
echo ""
echo ''
# ==============================================================================
# single PBS mode: one case per PBS job
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo "execute HAWC2, fork to background"
time WINEARCH=win32 WINEPREFIX=~/.wine32 wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp10_s100.htc &
echo 'execute HAWC2, fork to background'
time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp10_s100.htc' &
wait
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
echo "execute HAWC2, do not fork and wait"
time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp10_s100.htc
echo 'execute HAWC2, do not fork and wait'
(time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" numactl --physcpubind=$CPU_NR wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp10_s100.htc') 2>&1 | tee 'pbs_out/dlc01_demos/dlc01_steady_wsp10_s100.err.out'
fi
# ------------------------------------------------------------------------------
......@@ -80,51 +80,52 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### wait for jobs to finish
wait
echo ""
echo "Copy back from scratch directory"
cd /scratch/$USER/$PBS_JOBID/1/
mkdir -p $PBS_O_WORKDIR/res/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/logfiles/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/animation/
mkdir -p $PBS_O_WORKDIR/../turb/
cp -R res/dlc01_demos/. $PBS_O_WORKDIR/res/dlc01_demos/.
cp -R logfiles/dlc01_demos/. $PBS_O_WORKDIR/logfiles/dlc01_demos/.
cp -R animation/. $PBS_O_WORKDIR/animation/.
echo ''
echo 'Copy back from scratch directory'
cd "/scratch/$USER/$PBS_JOBID/1/"
mkdir -p "$PBS_O_WORKDIR/res/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/logfiles/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/animation/"
mkdir -p "$PBS_O_WORKDIR/../turb/"
cp -R 'res/dlc01_demos/.' "$PBS_O_WORKDIR/res/dlc01_demos/."
cp -R 'logfiles/dlc01_demos/.' "$PBS_O_WORKDIR/logfiles/dlc01_demos/."
cp -R 'animation/.' "$PBS_O_WORKDIR/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e $PBS_O_WORKDIR/../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i $PBS_O_WORKDIR/../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/1/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo ""
echo "following files are on node/cpu 1 (find .):"
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
echo ''
echo 'following files are on node/cpu 1 (find .):'
find .
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
rsync -a --remove-source-files res/dlc01_demos/. ../remote/res/dlc01_demos/.
rsync -a --remove-source-files logfiles/dlc01_demos/. ../remote/logfiles/dlc01_demos/.
rsync -a --remove-source-files animation/. ../remote/animation/.
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
rsync -a --remove-source-files 'res/dlc01_demos/.' "../remote/res/dlc01_demos/."
rsync -a --remove-source-files 'logfiles/dlc01_demos/.' "../remote/logfiles/dlc01_demos/."
rsync -a --remove-source-files 'pbs_out/dlc01_demos/.' "../remote/pbs_out/dlc01_demos/."
rsync -a --remove-source-files 'animation/.' "../remote/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e ../../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i ../../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
# ------------------------------------------------------------------------------
fi
exit
wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p
View file @ 62f0f2e4
### Standard Output
#PBS -N dlc01_steady_wsp11_s101
#PBS -o ./pbs_out/dlc01_demos/dlc01_steady_wsp11_s101.out
#PBS -o 'pbs_out/dlc01_demos/dlc01_steady_wsp11_s101.out'
### Standard Error
#PBS -e ./pbs_out/dlc01_demos/dlc01_steady_wsp11_s101.err
#PBS -e 'pbs_out/dlc01_demos/dlc01_steady_wsp11_s101.err'
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=04:00:00
......@@ -15,10 +15,10 @@
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### Create scratch directory and copy data to it
cd $PBS_O_WORKDIR
echo "current working dir (pwd):"
cd "$PBS_O_WORKDIR"
echo 'current working dir (pwd):'
pwd
cp -R ./demo_dlc_remote.zip /scratch/$USER/$PBS_JOBID
cp -R 'demo_dlc_remote.zip' "/scratch/$USER/$PBS_JOBID"
fi
# ==============================================================================
......@@ -29,20 +29,24 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo
echo 'Execute commands on scratch nodes'
cd /scratch/$USER/$PBS_JOBID
cd "/scratch/$USER/$PBS_JOBID"
# create unique dir for each CPU
mkdir "1"; cd "1"
mkdir '1'; cd '1'
pwd
/usr/bin/unzip ../demo_dlc_remote.zip
mkdir -p htc/dlc01_demos/
mkdir -p res/dlc01_demos/
mkdir -p logfiles/dlc01_demos/
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp11_s101.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/turb_s101_11ms*.bin turb/
mkdir -p 'htc/dlc01_demos/'
mkdir -p 'res/dlc01_demos/'
mkdir -p 'logfiles/dlc01_demos/'
mkdir -p 'turb/'
mkdir -p 'turb_micro/'
mkdir -p 'turb_meander/'
cp -R "$PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp11_s101.htc" 'htc/dlc01_demos/'
cp -R "$PBS_O_WORKDIR/../turb/turb_s101_11ms"*.bin "turb/"
cp -R "$PBS_O_WORKDIR/turb_micro/micro_s101_11ms"*.bin "turb_micro/"
cp -R "$PBS_O_WORKDIR/turb_meander/meander_s101_11ms"*.bin "turb_meander/"
_HOSTNAME_=`hostname`
if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
WINEARCH='win32' WINEPREFIX="$HOME/.wine32" winefix
fi
# ==============================================================================
......@@ -54,21 +58,21 @@ else
fi
# ------------------------------------------------------------------------------
echo ""
echo ''
# ==============================================================================
# single PBS mode: one case per PBS job
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo "execute HAWC2, fork to background"
time WINEARCH=win32 WINEPREFIX=~/.wine32 wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp11_s101.htc &
echo 'execute HAWC2, fork to background'
time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp11_s101.htc' &
wait
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
echo "execute HAWC2, do not fork and wait"
time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp11_s101.htc
echo 'execute HAWC2, do not fork and wait'
(time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" numactl --physcpubind=$CPU_NR wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp11_s101.htc') 2>&1 | tee 'pbs_out/dlc01_demos/dlc01_steady_wsp11_s101.err.out'
fi
# ------------------------------------------------------------------------------
......@@ -80,51 +84,66 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### wait for jobs to finish
wait
echo ""
echo "Copy back from scratch directory"
cd /scratch/$USER/$PBS_JOBID/1/
mkdir -p $PBS_O_WORKDIR/res/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/logfiles/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/animation/
mkdir -p $PBS_O_WORKDIR/../turb/
cp -R res/dlc01_demos/. $PBS_O_WORKDIR/res/dlc01_demos/.
cp -R logfiles/dlc01_demos/. $PBS_O_WORKDIR/logfiles/dlc01_demos/.
cp -R animation/. $PBS_O_WORKDIR/animation/.
echo ''
echo 'Copy back from scratch directory'
cd "/scratch/$USER/$PBS_JOBID/1/"
mkdir -p "$PBS_O_WORKDIR/res/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/logfiles/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/animation/"
mkdir -p "$PBS_O_WORKDIR/../turb/"
mkdir -p "$PBS_O_WORKDIR/turb_micro/"
mkdir -p "$PBS_O_WORKDIR/turb_meander/"
cp -R 'res/dlc01_demos/.' "$PBS_O_WORKDIR/res/dlc01_demos/."
cp -R 'logfiles/dlc01_demos/.' "$PBS_O_WORKDIR/logfiles/dlc01_demos/."
cp -R 'animation/.' "$PBS_O_WORKDIR/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e $PBS_O_WORKDIR/../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i $PBS_O_WORKDIR/../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/1/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
cd 'turb_micro/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/turb_micro/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/turb_micro/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
cd 'turb_meander/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/turb_meander/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/turb_meander/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo ""
echo "following files are on node/cpu 1 (find .):"
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
echo ''
echo 'following files are on node/cpu 1 (find .):'
find .
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
rsync -a --remove-source-files res/dlc01_demos/. ../remote/res/dlc01_demos/.
rsync -a --remove-source-files logfiles/dlc01_demos/. ../remote/logfiles/dlc01_demos/.
rsync -a --remove-source-files animation/. ../remote/animation/.
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
rsync -a --remove-source-files 'res/dlc01_demos/.' "../remote/res/dlc01_demos/."
rsync -a --remove-source-files 'logfiles/dlc01_demos/.' "../remote/logfiles/dlc01_demos/."
rsync -a --remove-source-files 'pbs_out/dlc01_demos/.' "../remote/pbs_out/dlc01_demos/."
rsync -a --remove-source-files 'animation/.' "../remote/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e ../../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i ../../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
cd 'turb_micro/'
for i in `ls *.bin`; do if [ -e "../remote/turb_micro/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "../remote/turb_micro/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
cd 'turb_meander/'
for i in `ls *.bin`; do if [ -e "../remote/turb_meander/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "../remote/turb_meander/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
# ------------------------------------------------------------------------------
fi
exit
wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p
View file @ 62f0f2e4
### Standard Output
#PBS -N dlc01_steady_wsp8_noturb
#PBS -o ./pbs_out/dlc01_demos/dlc01_steady_wsp8_noturb.out
#PBS -o 'pbs_out/dlc01_demos/dlc01_steady_wsp8_noturb.out'
### Standard Error
#PBS -e ./pbs_out/dlc01_demos/dlc01_steady_wsp8_noturb.err
#PBS -e 'pbs_out/dlc01_demos/dlc01_steady_wsp8_noturb.err'
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=04:00:00
......@@ -15,10 +15,10 @@
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### Create scratch directory and copy data to it
cd $PBS_O_WORKDIR
echo "current working dir (pwd):"
cd "$PBS_O_WORKDIR"
echo 'current working dir (pwd):'
pwd
cp -R ./demo_dlc_remote.zip /scratch/$USER/$PBS_JOBID
cp -R 'demo_dlc_remote.zip' "/scratch/$USER/$PBS_JOBID"
fi
# ==============================================================================
......@@ -29,20 +29,19 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo
echo 'Execute commands on scratch nodes'
cd /scratch/$USER/$PBS_JOBID
cd "/scratch/$USER/$PBS_JOBID"
# create unique dir for each CPU
mkdir "1"; cd "1"
mkdir '1'; cd '1'
pwd
/usr/bin/unzip ../demo_dlc_remote.zip
mkdir -p htc/dlc01_demos/
mkdir -p res/dlc01_demos/
mkdir -p logfiles/dlc01_demos/
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/none*.bin turb/
mkdir -p 'htc/dlc01_demos/'
mkdir -p 'res/dlc01_demos/'
mkdir -p 'logfiles/dlc01_demos/'
mkdir -p 'turb/'
cp -R "$PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc" 'htc/dlc01_demos/'
_HOSTNAME_=`hostname`
if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
WINEARCH='win32' WINEPREFIX="$HOME/.wine32" winefix
fi
# ==============================================================================
......@@ -54,21 +53,21 @@ else
fi
# ------------------------------------------------------------------------------
echo ""
echo ''
# ==============================================================================
# single PBS mode: one case per PBS job
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo "execute HAWC2, fork to background"
time WINEARCH=win32 WINEPREFIX=~/.wine32 wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc &
echo 'execute HAWC2, fork to background'
time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc' &
wait
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
echo "execute HAWC2, do not fork and wait"
time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc
echo 'execute HAWC2, do not fork and wait'
(time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" numactl --physcpubind=$CPU_NR wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc') 2>&1 | tee 'pbs_out/dlc01_demos/dlc01_steady_wsp8_noturb.err.out'
fi
# ------------------------------------------------------------------------------
......@@ -80,51 +79,52 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### wait for jobs to finish
wait
echo ""
echo "Copy back from scratch directory"
cd /scratch/$USER/$PBS_JOBID/1/
mkdir -p $PBS_O_WORKDIR/res/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/logfiles/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/animation/
mkdir -p $PBS_O_WORKDIR/../turb/
cp -R res/dlc01_demos/. $PBS_O_WORKDIR/res/dlc01_demos/.
cp -R logfiles/dlc01_demos/. $PBS_O_WORKDIR/logfiles/dlc01_demos/.
cp -R animation/. $PBS_O_WORKDIR/animation/.
echo ''
echo 'Copy back from scratch directory'
cd "/scratch/$USER/$PBS_JOBID/1/"
mkdir -p "$PBS_O_WORKDIR/res/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/logfiles/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/animation/"
mkdir -p "$PBS_O_WORKDIR/../turb/"
cp -R 'res/dlc01_demos/.' "$PBS_O_WORKDIR/res/dlc01_demos/."
cp -R 'logfiles/dlc01_demos/.' "$PBS_O_WORKDIR/logfiles/dlc01_demos/."
cp -R 'animation/.' "$PBS_O_WORKDIR/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e $PBS_O_WORKDIR/../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i $PBS_O_WORKDIR/../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/1/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo ""
echo "following files are on node/cpu 1 (find .):"
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
echo ''
echo 'following files are on node/cpu 1 (find .):'
find .
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
rsync -a --remove-source-files res/dlc01_demos/. ../remote/res/dlc01_demos/.
rsync -a --remove-source-files logfiles/dlc01_demos/. ../remote/logfiles/dlc01_demos/.
rsync -a --remove-source-files animation/. ../remote/animation/.
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
rsync -a --remove-source-files 'res/dlc01_demos/.' "../remote/res/dlc01_demos/."
rsync -a --remove-source-files 'logfiles/dlc01_demos/.' "../remote/logfiles/dlc01_demos/."
rsync -a --remove-source-files 'pbs_out/dlc01_demos/.' "../remote/pbs_out/dlc01_demos/."
rsync -a --remove-source-files 'animation/.' "../remote/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e ../../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i ../../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
# ------------------------------------------------------------------------------
fi
exit
wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p
View file @ 62f0f2e4
### Standard Output
#PBS -N dlc01_steady_wsp9_noturb
#PBS -o ./pbs_out/dlc01_demos/dlc01_steady_wsp9_noturb.out
#PBS -o 'pbs_out/dlc01_demos/dlc01_steady_wsp9_noturb.out'
### Standard Error
#PBS -e ./pbs_out/dlc01_demos/dlc01_steady_wsp9_noturb.err
#PBS -e 'pbs_out/dlc01_demos/dlc01_steady_wsp9_noturb.err'
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=04:00:00
......@@ -15,10 +15,10 @@
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### Create scratch directory and copy data to it
cd $PBS_O_WORKDIR
echo "current working dir (pwd):"
cd "$PBS_O_WORKDIR"
echo 'current working dir (pwd):'
pwd
cp -R ./demo_dlc_remote.zip /scratch/$USER/$PBS_JOBID
cp -R 'demo_dlc_remote.zip' "/scratch/$USER/$PBS_JOBID"
fi
# ==============================================================================
......@@ -29,20 +29,19 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo
echo 'Execute commands on scratch nodes'
cd /scratch/$USER/$PBS_JOBID
cd "/scratch/$USER/$PBS_JOBID"
# create unique dir for each CPU
mkdir "1"; cd "1"
mkdir '1'; cd '1'
pwd
/usr/bin/unzip ../demo_dlc_remote.zip
mkdir -p htc/dlc01_demos/
mkdir -p res/dlc01_demos/
mkdir -p logfiles/dlc01_demos/
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/none*.bin turb/
mkdir -p 'htc/dlc01_demos/'
mkdir -p 'res/dlc01_demos/'
mkdir -p 'logfiles/dlc01_demos/'
mkdir -p 'turb/'
cp -R "$PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc" 'htc/dlc01_demos/'
_HOSTNAME_=`hostname`
if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
WINEARCH='win32' WINEPREFIX="$HOME/.wine32" winefix
fi
# ==============================================================================
......@@ -54,21 +53,21 @@ else
fi
# ------------------------------------------------------------------------------
echo ""
echo ''
# ==============================================================================
# single PBS mode: one case per PBS job
# evaluates to true if LAUNCH_PBS_MODE is NOT set
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
echo "execute HAWC2, fork to background"
time WINEARCH=win32 WINEPREFIX=~/.wine32 wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc &
echo 'execute HAWC2, fork to background'
time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc' &
wait
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
echo "execute HAWC2, do not fork and wait"
time WINEARCH=win32 WINEPREFIX=~/.wine32 numactl --physcpubind=$CPU_NR wine hawc2-latest ./htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc
echo 'execute HAWC2, do not fork and wait'
(time WINEARCH='win32' WINEPREFIX="$HOME/.wine32" numactl --physcpubind=$CPU_NR wine 'hawc2-latest' 'htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc') 2>&1 | tee 'pbs_out/dlc01_demos/dlc01_steady_wsp9_noturb.err.out'
fi
# ------------------------------------------------------------------------------
......@@ -80,51 +79,52 @@ fi
if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
### wait for jobs to finish
wait
echo ""
echo "Copy back from scratch directory"
cd /scratch/$USER/$PBS_JOBID/1/
mkdir -p $PBS_O_WORKDIR/res/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/logfiles/dlc01_demos/
mkdir -p $PBS_O_WORKDIR/animation/
mkdir -p $PBS_O_WORKDIR/../turb/
cp -R res/dlc01_demos/. $PBS_O_WORKDIR/res/dlc01_demos/.
cp -R logfiles/dlc01_demos/. $PBS_O_WORKDIR/logfiles/dlc01_demos/.
cp -R animation/. $PBS_O_WORKDIR/animation/.
echo ''
echo 'Copy back from scratch directory'
cd "/scratch/$USER/$PBS_JOBID/1/"
mkdir -p "$PBS_O_WORKDIR/res/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/logfiles/dlc01_demos/"
mkdir -p "$PBS_O_WORKDIR/animation/"
mkdir -p "$PBS_O_WORKDIR/../turb/"
cp -R 'res/dlc01_demos/.' "$PBS_O_WORKDIR/res/dlc01_demos/."
cp -R 'logfiles/dlc01_demos/.' "$PBS_O_WORKDIR/logfiles/dlc01_demos/."
cp -R 'animation/.' "$PBS_O_WORKDIR/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e $PBS_O_WORKDIR/../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i $PBS_O_WORKDIR/../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/1/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/1/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo ""
echo "following files are on node/cpu 1 (find .):"
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
echo ''
echo 'following files are on node/cpu 1 (find .):'
find .
# ==============================================================================
# ------------------------------------------------------------------------------
# find+xargs mode: 1 PBS job, multiple cases
else
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
rsync -a --remove-source-files res/dlc01_demos/. ../remote/res/dlc01_demos/.
rsync -a --remove-source-files logfiles/dlc01_demos/. ../remote/logfiles/dlc01_demos/.
rsync -a --remove-source-files animation/. ../remote/animation/.
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
rsync -a --remove-source-files 'res/dlc01_demos/.' "../remote/res/dlc01_demos/."
rsync -a --remove-source-files 'logfiles/dlc01_demos/.' "../remote/logfiles/dlc01_demos/."
rsync -a --remove-source-files 'pbs_out/dlc01_demos/.' "../remote/pbs_out/dlc01_demos/."
rsync -a --remove-source-files 'animation/.' "../remote/animation/."
echo ""
echo "COPY BACK TURB IF APPLICABLE"
cd turb/
for i in `ls *.bin`; do if [ -e ../../turb/$i ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp $i ../../turb/; fi; done
cd /scratch/$USER/$PBS_JOBID/$CPU_NR/
echo "END COPY BACK TURB"
echo ""
echo ''
echo 'COPY BACK TURB IF APPLICABLE'
cd 'turb/'
for i in `ls *.bin`; do if [ -e "$PBS_O_WORKDIR/../turb/$i" ]; then echo "$i exists no copyback"; else echo "$i copyback"; cp "$i" "$PBS_O_WORKDIR/../turb/"; fi; done
cd "/scratch/$USER/$PBS_JOBID/$CPU_NR/"
echo 'END COPY BACK TURB'
echo ''
echo "COPYBACK [copyback_files]/[copyback_frename]"
echo "END COPYBACK"
echo ""
echo 'COPYBACK [copyback_files]/[copyback_frename]'
echo 'END COPYBACK'
echo ''
# ------------------------------------------------------------------------------
fi
exit
wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb_s100_10ms.p → wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb/turb_s100_10ms.p
View file @ 62f0f2e4
......@@ -35,7 +35,7 @@ echo "------------------------------------------------------------------------"
echo "EXECUTION"
echo "------------------------------------------------------------------------"
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe turb_s100_10ms 1.000000 29.400000 3.000000 100 8192 32 32 0.7812 6.5000 6.5000 1
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe 'turb_s100_10ms' 1.000000 29.400000 3.000000 100 8192 32 32 0.7812 6.5000 6.5000 1
### wait for jobs to finish
wait
......@@ -45,9 +45,9 @@ echo "CODA"
echo "------------------------------------------------------------------------"
# COPY BACK FROM SCRATCH AND RENAME, remove _ at end
cp turb_s100_10ms_u.bin $PBS_O_WORKDIR/../turb/turb_s100_10msu.bin
cp turb_s100_10ms_v.bin $PBS_O_WORKDIR/../turb/turb_s100_10msv.bin
cp turb_s100_10ms_w.bin $PBS_O_WORKDIR/../turb/turb_s100_10msw.bin
cp 'turb_s100_10ms_u.bin' "$PBS_O_WORKDIR/../turb/turb_s100_10msu.bin"
cp 'turb_s100_10ms_v.bin' "$PBS_O_WORKDIR/../turb/turb_s100_10msv.bin"
cp 'turb_s100_10ms_w.bin' "$PBS_O_WORKDIR/../turb/turb_s100_10msw.bin"
echo ""
......
wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb_s101_11ms.p → wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb/turb_s101_11ms.p
View file @ 62f0f2e4
......@@ -35,7 +35,7 @@ echo "------------------------------------------------------------------------"
echo "EXECUTION"
echo "------------------------------------------------------------------------"
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe turb_s101_11ms 1.000000 29.400000 3.000000 100 8192 32 32 0.8594 6.5000 6.5000 1
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe 'turb_s101_11ms' 1.000000 29.400000 3.000000 100 8192 32 32 0.8594 6.5000 6.5000 1
### wait for jobs to finish
wait
......@@ -45,9 +45,9 @@ echo "CODA"
echo "------------------------------------------------------------------------"
# COPY BACK FROM SCRATCH AND RENAME, remove _ at end
cp turb_s101_11ms_u.bin $PBS_O_WORKDIR/../turb/turb_s101_11msu.bin
cp turb_s101_11ms_v.bin $PBS_O_WORKDIR/../turb/turb_s101_11msv.bin
cp turb_s101_11ms_w.bin $PBS_O_WORKDIR/../turb/turb_s101_11msw.bin
cp 'turb_s101_11ms_u.bin' "$PBS_O_WORKDIR/../turb/turb_s101_11msu.bin"
cp 'turb_s101_11ms_v.bin' "$PBS_O_WORKDIR/../turb/turb_s101_11msv.bin"
cp 'turb_s101_11ms_w.bin' "$PBS_O_WORKDIR/../turb/turb_s101_11msw.bin"
echo ""
......
wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb_meander/meander_s101_11ms.p 0 → 100644
View file @ 62f0f2e4
### Standard Output
#PBS -N no_name_job
#PBS -o ./pbs_out/turb_meander/meander_s101_11ms.out
### Standard Error
#PBS -e ./pbs_out/turb_meander/meander_s101_11ms.err
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=00:59:59
#PBS -lnodes=1:ppn=1
### Queue name
#PBS -q workq
### #PBS -a [start_time]
### #PBS -W depend=afterany:[job_id]
### Browse to current working dir
echo ""
cd $PBS_O_WORKDIR
echo "current working dir:"
pwd
echo ""
### ===========================================================================
echo "------------------------------------------------------------------------"
echo "PRELUDE"
echo "------------------------------------------------------------------------"
winefix
cd /scratch/$USER/$PBS_JOBID/
echo ""
echo "------------------------------------------------------------------------"
echo "EXECUTION"
echo "------------------------------------------------------------------------"
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe 'meander_s101_11ms' 11.000000 12.000000 13.000000 101 9 9 50 0.9990 0.9990 1.3300 1
### wait for jobs to finish
wait
echo ""
echo "------------------------------------------------------------------------"
echo "CODA"
echo "------------------------------------------------------------------------"
# COPY BACK FROM SCRATCH AND RENAME, remove _ at end
cp 'meander_s101_11ms_u.bin' "$PBS_O_WORKDIR/turb_meander/meander_s101_11msu.bin"
cp 'meander_s101_11ms_v.bin' "$PBS_O_WORKDIR/turb_meander/meander_s101_11msv.bin"
cp 'meander_s101_11ms_w.bin' "$PBS_O_WORKDIR/turb_meander/meander_s101_11msw.bin"
echo ""
### ===========================================================================
exit
wetb/prepost/tests/data/demo_dlc/ref/pbs_in_turb/turb_micro/micro_s101_11ms.p 0 → 100644
View file @ 62f0f2e4
### Standard Output
#PBS -N no_name_job
#PBS -o ./pbs_out/turb_micro/micro_s101_11ms.out
### Standard Error
#PBS -e ./pbs_out/turb_micro/micro_s101_11ms.err
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=00:59:59
#PBS -lnodes=1:ppn=1
### Queue name
#PBS -q workq
### #PBS -a [start_time]
### #PBS -W depend=afterany:[job_id]
### Browse to current working dir
echo ""
cd $PBS_O_WORKDIR
echo "current working dir:"
pwd
echo ""
### ===========================================================================
echo "------------------------------------------------------------------------"
echo "PRELUDE"
echo "------------------------------------------------------------------------"
winefix
cd /scratch/$USER/$PBS_JOBID/
echo ""
echo "------------------------------------------------------------------------"
echo "EXECUTION"
echo "------------------------------------------------------------------------"
time WINEARCH=win64 WINEPREFIX=~/.wine wine mann_turb_x64.exe 'micro_s101_11ms' 21.000000 22.000000 23.000000 101 19 19 60 10.9990 10.9990 11.3300 1
### wait for jobs to finish
wait
echo ""
echo "------------------------------------------------------------------------"
echo "CODA"
echo "------------------------------------------------------------------------"
# COPY BACK FROM SCRATCH AND RENAME, remove _ at end
cp 'micro_s101_11ms_u.bin' "$PBS_O_WORKDIR/turb_micro/micro_s101_11msu.bin"
cp 'micro_s101_11ms_v.bin' "$PBS_O_WORKDIR/turb_micro/micro_s101_11msv.bin"
cp 'micro_s101_11ms_w.bin' "$PBS_O_WORKDIR/turb_micro/micro_s101_11msw.bin"
echo ""
### ===========================================================================
exit
wetb/prepost/tests/data/demo_dlc/ref/prepost/remote_tags.txt
View file @ 62f0f2e4
......@@ -25,6 +25,12 @@
[G_t0] : True
[Grid loss time] : 5000
[Induction] : 1
[MannAlfaEpsilon_meander] : True
[MannAlfaEpsilon_micro] : True
[MannGamma_meander] : True
[MannGamma_micro] : True
[MannL_meander] : True
[MannL_micro] : True
[Pitch 1 DLC22b] : 0
[Pitvel 1] : 3
[Pitvel 2] : 4
......@@ -34,7 +40,7 @@
[TI] : 0.232
[Time pitch runaway] : 5000
[Time stuck DLC22b] : -1
[Turb base name] : none
[Turb base name] : None
[Windspeed] : 8
[case_id] : dlc01_steady_wsp8_noturb
[data_dir] : data/
......@@ -42,15 +48,23 @@
[duration] : 20.0
[gust] : False
[gust_type] : True
[high_freq_comp_meander] : True
[high_freq_comp_micro] : True
[htc_dir] : htc/dlc01_demos/
[init_wr] : 0.5
[iter_dir] : iter/dlc01_demos/
[log_dir] : logfiles/dlc01_demos/
[meander_base_name] : None
[meander_dir] : None
[micro_base_name] : None
[micro_dir] : None
[out_format] : hawc_binary
[pbs_in_dir] : pbs_in/dlc01_demos/
[pbs_out_dir] : pbs_out/dlc01_demos/
[res_dir] : res/dlc01_demos/
[seed] : 0
[seed_meander] : True
[seed_micro] : True
[shear_exp] : 0
[staircase] : False
[t flap on] : -1
......@@ -59,8 +73,21 @@
[time_stop] : 40
[tu_model] : 0
[tu_seed] : 0
[turb_base_name] : none
[turb_dx] : 0.03906
[turb_base_name] : None
[turb_dir] : turb/
[turb_dx] : 0.0390625
[turb_dx_meander] : True
[turb_dx_micro] : True
[turb_dy_meander] : True
[turb_dy_micro] : True
[turb_dz_meander] : True
[turb_dz_micro] : True
[turb_nr_u_meander] : True
[turb_nr_u_micro] : True
[turb_nr_v_meander] : True
[turb_nr_v_micro] : True
[turb_nr_w_meander] : True
[turb_nr_w_micro] : True
[wdir] : 0
[windramp] : False
[wsp factor] : 1.0
......@@ -83,6 +110,12 @@
[G_t0] : True
[Grid loss time] : 5000
[Induction] : 1
[MannAlfaEpsilon_meander] : True
[MannAlfaEpsilon_micro] : True
[MannGamma_meander] : True
[MannGamma_micro] : True
[MannL_meander] : True
[MannL_micro] : True
[Pitch 1 DLC22b] : 0
[Pitvel 1] : 3
[Pitvel 2] : 4
......@@ -92,7 +125,7 @@
[TI] : 0.2196
[Time pitch runaway] : 5000
[Time stuck DLC22b] : -1
[Turb base name] : none
[Turb base name] : None
[Windspeed] : 9
[case_id] : dlc01_steady_wsp9_noturb
[data_dir] : data/
......@@ -100,15 +133,23 @@
[duration] : 20.0
[gust] : False
[gust_type] : True
[high_freq_comp_meander] : True
[high_freq_comp_micro] : True
[htc_dir] : htc/dlc01_demos/
[init_wr] : 0.5
[iter_dir] : iter/dlc01_demos/
[log_dir] : logfiles/dlc01_demos/
[meander_base_name] : None
[meander_dir] : None
[micro_base_name] : None
[micro_dir] : None
[out_format] : hawc_binary
[pbs_in_dir] : pbs_in/dlc01_demos/
[pbs_out_dir] : pbs_out/dlc01_demos/
[res_dir] : res/dlc01_demos/
[seed] : 0
[seed_meander] : True
[seed_micro] : True
[shear_exp] : 0
[staircase] : False
[t flap on] : -1
......@@ -117,8 +158,21 @@
[time_stop] : 40
[tu_model] : 0
[tu_seed] : 0
[turb_base_name] : none
[turb_dx] : 0.04395
[turb_base_name] : None
[turb_dir] : turb/
[turb_dx] : 0.0439453
[turb_dx_meander] : True
[turb_dx_micro] : True
[turb_dy_meander] : True
[turb_dy_micro] : True
[turb_dz_meander] : True
[turb_dz_micro] : True
[turb_nr_u_meander] : True
[turb_nr_u_micro] : True
[turb_nr_v_meander] : True
[turb_nr_v_micro] : True
[turb_nr_w_meander] : True
[turb_nr_w_micro] : True
[wdir] : 0
[windramp] : False
[wsp factor] : 0.8889
......@@ -141,6 +195,12 @@
[G_t0] : True
[Grid loss time] : 5000
[Induction] : 1
[MannAlfaEpsilon_meander] : True
[MannAlfaEpsilon_micro] : True
[MannGamma_meander] : True
[MannGamma_micro] : True
[MannL_meander] : True
[MannL_micro] : True
[Pitch 1 DLC22b] : 0
[Pitvel 1] : 3
[Pitvel 2] : 4
......@@ -158,15 +218,23 @@
[duration] : 20.0
[gust] : False
[gust_type] : True
[high_freq_comp_meander] : True
[high_freq_comp_micro] : True
[htc_dir] : htc/dlc01_demos/
[init_wr] : 0.5
[iter_dir] : iter/dlc01_demos/
[log_dir] : logfiles/dlc01_demos/
[meander_base_name] : None
[meander_dir] : None
[micro_base_name] : None
[micro_dir] : None
[out_format] : hawc_binary
[pbs_in_dir] : pbs_in/dlc01_demos/
[pbs_out_dir] : pbs_out/dlc01_demos/
[res_dir] : res/dlc01_demos/
[seed] : 100
[seed_meander] : True
[seed_micro] : True
[shear_exp] : 0
[staircase] : False
[t flap on] : -1
......@@ -176,7 +244,20 @@
[tu_model] : 1
[tu_seed] : 100
[turb_base_name] : turb_s100_10ms
[turb_dir] : turb/
[turb_dx] : 0.78125
[turb_dx_meander] : True
[turb_dx_micro] : True
[turb_dy_meander] : True
[turb_dy_micro] : True
[turb_dz_meander] : True
[turb_dz_micro] : True
[turb_nr_u_meander] : True
[turb_nr_u_micro] : True
[turb_nr_v_meander] : True
[turb_nr_v_micro] : True
[turb_nr_w_meander] : True
[turb_nr_w_micro] : True
[wdir] : 0
[windramp] : False
[wsp factor] : 0.8
......@@ -199,6 +280,12 @@
[G_t0] : True
[Grid loss time] : 5000
[Induction] : 1
[MannAlfaEpsilon_meander] : 11.0
[MannAlfaEpsilon_micro] : 21.0
[MannGamma_meander] : 13.0
[MannGamma_micro] : 23.0
[MannL_meander] : 12.0
[MannL_micro] : 22.0
[Pitch 1 DLC22b] : 0
[Pitvel 1] : 3
[Pitvel 2] : 4
......@@ -216,15 +303,23 @@
[duration] : 20.0
[gust] : False
[gust_type] : True
[high_freq_comp_meander] : 1.0
[high_freq_comp_micro] : True
[htc_dir] : htc/dlc01_demos/
[init_wr] : 0.5
[iter_dir] : iter/dlc01_demos/
[log_dir] : logfiles/dlc01_demos/
[meander_base_name] : meander_s101_11ms
[meander_dir] : turb_meander/
[micro_base_name] : micro_s101_11ms
[micro_dir] : turb_micro/
[out_format] : hawc_binary
[pbs_in_dir] : pbs_in/dlc01_demos/
[pbs_out_dir] : pbs_out/dlc01_demos/
[res_dir] : res/dlc01_demos/
[seed] : 100
[seed_meander] : 101.0
[seed_micro] : 101.0
[shear_exp] : 0
[staircase] : False
[t flap on] : -1
......@@ -234,7 +329,20 @@
[tu_model] : 1
[tu_seed] : 100
[turb_base_name] : turb_s101_11ms
[turb_dx] : 0.85938
[turb_dir] : turb/
[turb_dx] : 0.859375
[turb_dx_meander] : 0.999
[turb_dx_micro] : 10.999
[turb_dy_meander] : 0.999
[turb_dy_micro] : 10.999
[turb_dz_meander] : 1.33
[turb_dz_micro] : 11.33
[turb_nr_u_meander] : 9.0
[turb_nr_u_micro] : 19.0
[turb_nr_v_meander] : 9.0
[turb_nr_v_micro] : 19.0
[turb_nr_w_meander] : 50.0
[turb_nr_w_micro] : 60.0
[wdir] : 0
[windramp] : False
[wsp factor] : 0.7273
......
wetb/prepost/tests/data/demo_dlc/ref/zip-chunks-gorm/remote_chnk_00000.p deleted 100644 → 0
View file @ 64a92f7c
### Standard Output
#PBS -N remote_chnk_00000
#PBS -o ./pbs_out_chunks/remote_chnk_00000.out
### Standard Error
#PBS -e ./pbs_out_chunks/remote_chnk_00000.err
#PBS -W umask=0003
### Maximum wallclock time format HOURS:MINUTES:SECONDS
#PBS -l walltime=20:00:00
#PBS -l nodes=1:ppn=12
### Queue name
#PBS -q workq
echo "----------------------------------------------------------------------"
echo "activate python environment wetb_py3"
source /home/python/miniconda3/bin/activate wetb_py3
echo "CHECK 2x IF wetb_py3 IS ACTIVE, IF NOT TRY AGAIN"
CMD="from distutils.sysconfig import get_python_lib;print (get_python_lib().find('wetb_py3'))"
ACTIVATED=`python -c "$CMD"`
if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi
ACTIVATED=`python -c "$CMD"`
if [ $ACTIVATED -eq -1 ]; then source activate wetb_py3;fi
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/
echo 'current working directory:'
pwd
echo "create CPU directories on the scratch disk"
mkdir -p /scratch/$USER/$PBS_JOBID/remote/
mkdir -p /scratch/$USER/$PBS_JOBID/0/
mkdir -p /scratch/$USER/$PBS_JOBID/1/
mkdir -p /scratch/$USER/$PBS_JOBID/2/
mkdir -p /scratch/$USER/$PBS_JOBID/3/
mkdir -p /scratch/$USER/$PBS_JOBID/4/
mkdir -p /scratch/$USER/$PBS_JOBID/5/
mkdir -p /scratch/$USER/$PBS_JOBID/6/
mkdir -p /scratch/$USER/$PBS_JOBID/7/
mkdir -p /scratch/$USER/$PBS_JOBID/8/
mkdir -p /scratch/$USER/$PBS_JOBID/9/
mkdir -p /scratch/$USER/$PBS_JOBID/10/
mkdir -p /scratch/$USER/$PBS_JOBID/11/
echo "----------------------------------------------------------------------"
cd $PBS_O_WORKDIR
echo 'current working directory:'
pwd
echo "get the zip-chunk file from the PBS_O_WORKDIR"
cp ./zip-chunks-gorm/remote_chnk_00000.zip /scratch/$USER/$PBS_JOBID/
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/
echo 'current working directory:'
pwd
echo "unzip chunk, create dirs in cpu and sim_id folders"
/usr/bin/unzip remote_chnk_00000.zip -d 0/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 1/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 2/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 3/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 4/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 5/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 6/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 7/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 8/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 9/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 10/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d 11/. >> /dev/null
/usr/bin/unzip remote_chnk_00000.zip -d remote/. >> /dev/null
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/remote/
echo 'current working directory:'
pwd
echo "create turb_db directories"
mkdir -p ../turb/
echo "----------------------------------------------------------------------"
cd $PBS_O_WORKDIR
echo 'current working directory:'
pwd
# copy to scratch db directory for [turb_db_dir], [turb_base_name]
cp ../turb/none* /scratch/$USER/$PBS_JOBID/remote/../turb/.
cp ../turb/turb_s100_10ms* /scratch/$USER/$PBS_JOBID/remote/../turb/.
cp ../turb/turb_s101_11ms* /scratch/$USER/$PBS_JOBID/remote/../turb/.
# copy to scratch db directory for [meand_db_dir], [meand_base_name]
# copy to scratch db directory for [wake_db_dir], [wake_base_name]
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/remote/
echo 'current working directory:'
pwd
echo "create turb directories in CPU dirs"
mkdir -p 0/turb/
mkdir -p 1/turb/
mkdir -p 2/turb/
mkdir -p 3/turb/
mkdir -p 4/turb/
mkdir -p 5/turb/
mkdir -p 6/turb/
mkdir -p 7/turb/
mkdir -p 8/turb/
mkdir -p 9/turb/
mkdir -p 10/turb/
mkdir -p 11/turb/
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/remote/
echo 'current working directory:'
pwd
echo "Link all turb files into CPU dirs"
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/0/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/1/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/2/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/3/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/4/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/5/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/6/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/7/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/8/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/9/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/10/turb/ \;
find /scratch/$USER/$PBS_JOBID/../turb/ -iname "*.bin" -exec ln -s {} /scratch/$USER/$PBS_JOBID/remote/11/turb/ \;
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/
echo 'current working directory:'
pwd
echo "START RUNNING JOBS IN find+xargs MODE"
WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
# run all the PBS *.p files in find+xargs mode
echo "following cases will be run from following path:"
echo "remote/pbs_in/dlc01_demos/"
export LAUNCH_PBS_MODE=false
/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' | sort -z
echo "number of files to be launched: "`find remote/pbs_in/dlc01_demos/ -type f | wc -l`
/home/MET/sysalt/bin/find remote/pbs_in/dlc01_demos/ -type f -name '*.p' -print0 | sort -z | /home/MET/sysalt/bin/xargs -0 -I{} --process-slot-var=CPU_NR -n 1 -P 12 sh {}
echo "END OF JOBS IN find+xargs MODE"
echo "----------------------------------------------------------------------"
echo 'total scratch disk usage:'
du -hs /scratch/$USER/$PBS_JOBID/
cd /scratch/$USER/$PBS_JOBID/remote
echo 'current working directory:'
pwd
echo "Results saved at sim_id directory:"
find
echo "move statsdel into compressed archive"
find res/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/statsdel_chnk_00000.tar
xz -z2 -T 12 prepost/statsdel_chnk_00000.tar
echo "move log analysis into compressed archive"
find logfiles/dlc01_demos/ -name "*.csv" -print0 | xargs -0 tar --remove-files -rf prepost/loganalysis_chnk_00000.tar
xz -z2 -T 12 prepost/loganalysis_chnk_00000.tar
echo "----------------------------------------------------------------------"
cd /scratch/$USER/$PBS_JOBID/
echo 'current working directory:'
pwd
echo "move results back from node scratch/sim_id to origin, but ignore htc, and pbs_in directories."
echo "copy from remote/* to $PBS_O_WORKDIR/"
time rsync -au --remove-source-files remote/* $PBS_O_WORKDIR/ \
--exclude pbs_in/dlc01_demos/* \
--exclude *.htc
source deactivate
echo "DONE !!"
echo "----------------------------------------------------------------------"
# in case wine has crashed, kill any remaining wine servers
# caution: ALL the users wineservers will die on this node!
echo "following wineservers are still running:"
ps -u $USER -U $USER | grep wineserver
killall -u $USER wineserver
exit