diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 307784f8f8a750aabb68dd96c0239d5553084404..74147a94fe67733d80d4035bbf6c5b4b15670bea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,4 +4,7 @@ before_script:
test-3.4:
image: mmpe/wetb
script:
- - python3 setup.py test
\ No newline at end of file
+ #- python3 setup.py test
+ - pip3 install pytest-cov
+ - python3 -m pytest --cov=wetb
+
diff --git a/README.md b/README.md
index b8f2fbbb678a74baacf870f4f16355e27f5c5cc1..f019bc546495292a060e55d267991173a96ff24f 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,6 @@
+[](https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/commits/master)
+[](https://gitlab.windenergy.dtu.dk/toolbox/WindEnergyToolbox/commits/master)
# Introduction
The Wind Energy Toolbox (or ```wetb```, pronounce as wee-tee-bee) is a collection
diff --git a/docs/configure-wine.md b/docs/configure-wine.md
index e2c891bceff0694989144b9cfbc17dda6884e8f2..31b2a222b173aee5c9aca65e1a98df12be5e67a2 100644
--- a/docs/configure-wine.md
+++ b/docs/configure-wine.md
@@ -1,5 +1,5 @@
-Configure Wine for Gorm
-------------------------
+Background Information Regarding Wine
+-------------------------------------
> Note that the steps described here are executed automatically by the
configuration script [```config-wine-hawc2.sh```]
@@ -7,6 +7,9 @@ configuration script [```config-wine-hawc2.sh```]
in ```pbsutils```.
+Configure Wine for Gorm
+------------------------
+
You will also need to configure wine and place the HAWC2 executables in a
directory that wine knows about. First, activate the correct wine environment by
typing in a shell in the Gorm's home directory (it can be activated with
@@ -24,23 +27,24 @@ command). In the ```.bashrc``` file in your home directory
alias wine32='WINEARCH=win32 WINEPREFIX=~/.wine32 wine'
```
-Add a folder called ```bin``` to your wine system's PATH so we can copy all
-the HAWC2 executables in here:
+Add a folder called ```~/wine_exe/win32``` to your wine system's PATH so we can
+copy all the HAWC2 executables in here:
```
-WINEDIRNAME=".wine32"
-printf 'REGEDIT4\n[HKEY_CURRENT_USER\\Environment]\n"PATH"="c:\\bin"' >> ./tmp.reg
-WINEARCH=win32 WINEPREFIX=~/$WINEDIRNAME wine regedit ./tmp.reg
+$EXE_DIR_WINE="z:/home/$USER/wine_exe/win32/"
+printf 'REGEDIT4\n[HKEY_CURRENT_USER\\Environment]\n"PATH"="'"$EXE_DIR_WINE"'"\n' >> ./tmp.reg
+WINEARCH=win32 WINEPREFIX=~/.wine32 wine regedit ./tmp.reg
rm ./tmp.reg
```
And now copy all the HAWC2 executables, DLL's (including the license manager)
to your wine directory. You can copy all the required executables, dll's and
the license manager are located at ```/home/MET/hawc2exe```. The following
-command will do this copying:
+command will update your local directory with any new executables that have
+been placed in ```/home/MET/hawc2exe/win32/```:
```
-g-000 $ cp /home/MET/hawc2exe/* /home/$USER/.wine32/drive_c/bin/
+g-000 $ rsync -a /home/MET/hawc2exe/win32/* /home/$USER/wine_exe/win32/
```
Notice that the HAWC2 executable names are ```hawc2-latest.exe```,
@@ -55,7 +59,19 @@ simply copy all the files from there again to update.
Configure Wine for Jess
------------------------
+Same principles apply to Jess, and [```config-wine-hawc2.sh```]
+(https://gitlab.windenergy.dtu.dk/toolbox/pbsutils/blob/master/config-wine-hawc2.sh)
+can be used to initialize and configure your wine environment.
+
+Note that due to a bug in the specific version of wine that is installed on
+Jess, ```config-wine-hawc2.sh``` will apply the following command to fix this.
+It is important to note that this fix will have to be executed on each node at
+the beginning of each new session:
+
```
-g-000 $ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
-g-000 $ WINEARCH=win32 WINEPREFIX=~/.wine32 wine test.exe
+j-000 $ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
```
+
+```winefix``` is automatically included in the ```pbs_in``` scripts genetrated
+by the toolbox.
+
diff --git a/setup.cfg b/setup.cfg
index d2a646f0bdebeeaa4cde8ce65d6fe06d26160521..6c20abb966dd98029c40dac0f33ffcc90a89b7b6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -62,6 +62,7 @@ packages =
#addopts =
# --cov wetb --cov-report term-missing
# --verbose
+python_files = WindEnergyToolbox/wetb/*
[aliases]
docs = build_sphinx
diff --git a/wetb/fast/tests/__init__.py b/wetb/fast/tests/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/wetb/fatigue_tools/rainflowcounting/compile.py b/wetb/fatigue_tools/rainflowcounting/compile.py
deleted file mode 100644
index ba4576622a7bb3bfc76fbef7c76379d63cf64ed5..0000000000000000000000000000000000000000
--- a/wetb/fatigue_tools/rainflowcounting/compile.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from __future__ import unicode_literals
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from wetb.utils.cython_compile.cython_compile import cython_import
-cython_import('pair_range')
-cython_import('peak_trough')
-cython_import('rainflowcount_astm')
diff --git a/wetb/fatigue_tools/rainflowcounting/rainflowcount.py b/wetb/fatigue_tools/rainflowcounting/rainflowcount.py
index 645da85e332a8cd5296d602e982c46b1cd79c1cf..8d1bdd76e8bb550114bb552e5ae6985a8c9b427b 100644
--- a/wetb/fatigue_tools/rainflowcounting/rainflowcount.py
+++ b/wetb/fatigue_tools/rainflowcounting/rainflowcount.py
@@ -6,7 +6,7 @@ from builtins import str
from future import standard_library
standard_library.install_aliases()
import numpy as np
-from wetb.utils.cython_compile.cython_compile import cython_import
+
def check_signal(signal):
@@ -73,9 +73,6 @@ def rainflow_windap(signal, levels=255., thresshold=(255 / 50)):
# If possible the module is compiled using cython otherwise the python implementation is used
- cython_import('wetb.fatigue_tools.rainflowcounting.peak_trough')
- cython_import('wetb.fatigue_tools.rainflowcounting.pair_range')
-
from wetb.fatigue_tools.rainflowcounting.peak_trough import peak_trough
from wetb.fatigue_tools.rainflowcounting.pair_range import pair_range_amplitude_mean
@@ -129,7 +126,7 @@ def rainflow_astm(signal):
# Import find extremes and rainflow.
# If possible the module is compiled using cython otherwise the python implementation is used
- cython_import('wetb.fatigue_tools.rainflowcounting.rainflowcount_astm')
+
from wetb.fatigue_tools.rainflowcounting.rainflowcount_astm import find_extremes, rainflowcount
# Remove points which is not local minimum/maximum
diff --git a/wetb/gtsdf/__init__.py b/wetb/gtsdf/__init__.py
index 7339aa3425109db20313635013bf75b0e14db84c..364d20a40cbaa101738ccc39ca6e896806f2ab1a 100644
--- a/wetb/gtsdf/__init__.py
+++ b/wetb/gtsdf/__init__.py
@@ -44,12 +44,25 @@ class Dataset(object):
self.time, self.data, self.info = load(filename)
def __call__(self, id):
if isinstance(id, str):
- return self((['Time'] + self.info['attribute_names']).index(id) + 1)
+ if id=="Time":
+ return self.time
+ else:
+ return self(self.info['attribute_names'].index(id) + 2)
if id == 1:
return self.time
else:
return self.data[:, id - 2]
+ def __getattribute__(self, name):
+ try:
+ return object.__getattribute__(self, name)
+
+ except Exception as e:
+ try:
+ return self(name)
+ except:
+ raise e
+
__all__ = sorted([m for m in set(dir()) - set(d)])
diff --git a/wetb/gtsdf/tests/test_gtsdf.py b/wetb/gtsdf/tests/test_gtsdf.py
index 004eed321612f21ae8dd045db9e13f87d7a7fb99..7a5ed608a2918223edfbdda3a010b17798ca8058 100644
--- a/wetb/gtsdf/tests/test_gtsdf.py
+++ b/wetb/gtsdf/tests/test_gtsdf.py
@@ -18,10 +18,10 @@ from wetb import gtsdf
import unittest
import os
-import shutil
tmp_path = os.path.dirname(__file__) + "/tmp/"
-tfp = os.path.dirname(__file__) + "/test_files/"
+tfp = os.path.dirname(os.path.abspath(__file__)) + "/test_files/"
+
class Test_gsdf(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
@@ -230,11 +230,33 @@ class Test_gsdf(unittest.TestCase):
df = gtsdf.load_pandas(fn)
def test_loadtesthdf5(self):
-
time, data, info = gtsdf.load(tfp + 'test.hdf5')
self.assertEqual(time[1], 0.05)
self.assertEqual(data[1, 1], 11.986652374267578)
self.assertEqual(info['attribute_names'][1], "WSP gl. coo.,Vy")
+
+ def test_loadhdf5File(self):
+ f = h5py.File(tfp + 'test.hdf5')
+ time, data, info = gtsdf.load(f)
+
+ self.assertEqual(time[1], 0.05)
+ self.assertEqual(data[1, 1], 11.986652374267578)
+ self.assertEqual(info['attribute_names'][1], "WSP gl. coo.,Vy")
+
+
+ def test_gtsdf_dataset(self):
+ ds = gtsdf.Dataset(tfp+'test.hdf5')
+ self.assertEqual(ds.data.shape, (2440,49))
+ self.assertEqual(ds('Time')[1], 0.05)
+ self.assertEqual(ds.Time[1], 0.05)
+ self.assertRaisesRegex(AttributeError, "'Dataset' object has no attribute 'Time1'", lambda : ds.Time1)
+ self.assertEqual(ds(2)[1], 12.04148006439209)
+ n = ds.info['attribute_names'][2]
+ self.assertEqual(n, "WSP gl. coo.,Vy")
+ self.assertEqual(ds(n)[1], 12.04148006439209)
+
+
+
diff --git a/wetb/gtsdf/tests/test_unix_time.py b/wetb/gtsdf/tests/test_unix_time.py
index 434b7c3d87446840831aadf273943f2476604b07..f976195a1dcedd22b82fc28d35d25e8b2ddf4f87 100644
--- a/wetb/gtsdf/tests/test_unix_time.py
+++ b/wetb/gtsdf/tests/test_unix_time.py
@@ -12,30 +12,33 @@ standard_library.install_aliases()
import unittest
import numpy as np
import datetime
-from wetb.utils.timing import print_time
from wetb.gtsdf.unix_time import to_unix, from_unix
+
class TestUnixTime(unittest.TestCase):
def test_to_unix(self):
+ print (np.array([5]))
self.assertEqual(to_unix(datetime.datetime(2016, 2, 2, 13, 6, 25)), 1454418385)
+ self.assertEqual(to_unix([datetime.datetime(2016, 2, 2, 13, 6, 25),datetime.datetime(2016, 2, 2, 13, 6, 26)]), [1454418385,1454418386])
self.assertNotEqual(to_unix(datetime.datetime(2016, 2, 2, 13, 6, 26)), 1454418385)
+ self.assertRaises(Exception, to_unix,1)
+
+
+
+
def test_from_unix(self):
self.assertEqual(from_unix(1454418385), datetime.datetime(2016, 2, 2, 13, 6, 25))
self.assertNotEqual(from_unix(1454418385), datetime.datetime(2016, 2, 2, 13, 6, 26))
+
+ self.assertEqual(from_unix(np.nan), datetime.datetime(1970,1,1,0,0))
+ self.assertEqual(from_unix([1454418385,1454418386]), [datetime.datetime(2016, 2, 2, 13, 6, 25),datetime.datetime(2016, 2, 2, 13, 6, 26)])
+
-# @print_time
-# def r(self, dt):
-# return [to_unix(dt) for dt in dt]
-
-# def test_to_unix_time(self):
-# dt = [datetime.datetime(2000, 1, 1, 12, s % 60) for s in np.arange(1000000)]
-# self.r(dt)
-
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
unittest.main()
diff --git a/wetb/gtsdf/unix_time.py b/wetb/gtsdf/unix_time.py
index dfd4bc6ebb2beadd4720cecf292ab84d199855e3..c5ac88c06e9f8978e8415574f5c74f80bff9ccae 100644
--- a/wetb/gtsdf/unix_time.py
+++ b/wetb/gtsdf/unix_time.py
@@ -18,10 +18,10 @@ def to_unix(dateTime):
return [(dt - timestamp0).total_seconds() for dt in dateTime]
raise
-def from_unix_old(sec):
- if np.isnan(sec):
- return datetime.utcfromtimestamp(0)
- return datetime.utcfromtimestamp(sec)
+# def from_unix_old(sec):
+# if np.isnan(sec):
+# return datetime.utcfromtimestamp(0)
+# return datetime.utcfromtimestamp(sec)
day_dict = {}
diff --git a/wetb/hawc2/ascii2bin/Example.py b/wetb/hawc2/ascii2bin/Example.py
index 38785246e1e7a920711ce2ad2fbfdd21ec8378bf..b0afee5e9af2d676c50a77d3618534446f18c337 100644
--- a/wetb/hawc2/ascii2bin/Example.py
+++ b/wetb/hawc2/ascii2bin/Example.py
@@ -11,6 +11,6 @@ from future import standard_library
standard_library.install_aliases()
from wetb.hawc2.ascii2bin.ascii2bin import ascii2bin
-
-ascii2bin(r"tests/hawc2ascii.sel", "temp_hawc2ascii.sel")
+if __name__=="__main__":
+ ascii2bin(r"tests/hawc2ascii.sel", "temp_hawc2ascii.sel")
diff --git a/wetb/hawc2/ascii2bin/make_exe.py b/wetb/hawc2/ascii2bin/make_exe.py
index 7e31b83af2c8607331179e01e6b267da5b002daa..8b1e848075529830a94f1595d1fb5fb2332061a3 100644
--- a/wetb/hawc2/ascii2bin/make_exe.py
+++ b/wetb/hawc2/ascii2bin/make_exe.py
@@ -6,11 +6,9 @@ Created on 06/09/2013
from __future__ import division, print_function, absolute_import, unicode_literals
from future import standard_library
standard_library.install_aliases()
-from build_exe.cx.build_cx_exe import NUMPY
-try: range = xrange; xrange = None
-except NameError: pass
-try: str = unicode; unicode = None
-except NameError: pass
-from build_exe.cx import build_cx_exe
-build_cx_exe.build_exe("ascii2bin.py", version="3.0.1", includes=["'pandas'"], modules=['email', NUMPY])
+
+if __name__=="__main__":
+ from build_exe.cx.build_cx_exe import NUMPY
+ from build_exe.cx import build_cx_exe
+ build_cx_exe.build_exe("ascii2bin.py", version="3.0.1", includes=["'pandas'"], modules=['email', NUMPY])
diff --git a/wetb/hawc2/simulation.py b/wetb/hawc2/simulation.py
index c25b08074eed66d94fb3778c5831a66024e081f8..01ae9412adf0d6484edc853c88a334f862abaf83 100755
--- a/wetb/hawc2/simulation.py
+++ b/wetb/hawc2/simulation.py
@@ -86,7 +86,7 @@ class Simulation(object):
if not os.path.isabs(htcfilename):
htcfilename = os.path.join(modelpath, htcfilename)
self.filename = os.path.basename(htcfilename)
- self.htcFile = HTCFile(htcfilename)
+ self.htcFile = HTCFile(htcfilename, os.path.relpath(self.modelpath,os.path.dirname(htcfilename)))
self.time_stop = self.htcFile.simulation.time_stop[0]
self.hawc2exe = hawc2exe
self.copy_turbulence = copy_turbulence
@@ -127,7 +127,7 @@ class Simulation(object):
def abort(self, update_status=True):
if self.status != QUEUED:
self.host.stop()
- for _ in range(100):
+ for _ in range(50):
if self.is_simulating is False:
break
time.sleep(0.1)
@@ -175,7 +175,7 @@ class Simulation(object):
src = os.path.relpath (src)
assert not src.startswith(".."), "%s referes to a file outside the model path\nAll input files be inside model path" % src
return src
- input_patterns = [fmt(src) for src in self.htcFile.input_files() + self.htcFile.turbulence_files() + self.additional_files().get('input', [])]
+ input_patterns = [fmt(src) for src in self.htcFile.input_files() + ([], self.htcFile.turbulence_files())[self.copy_turbulence] + self.additional_files().get('input', [])]
input_files = set([f for pattern in input_patterns for f in glob.glob(os.path.join(self.modelpath, pattern)) if os.path.isfile(f)])
if not os.path.isdir(os.path.dirname(self.modelpath + self.stdout_filename)):
os.makedirs(os.path.dirname(self.modelpath + self.stdout_filename))
@@ -435,7 +435,6 @@ class LocalSimulationHost(SimulationResource):
self.logFile.update_status()
def stop(self):
- if self.simulationThread.is_alive():
self.simulationThread.stop()
self.simulationThread.join()
@@ -459,27 +458,15 @@ class SimulationThread(Thread):
stdout = self.sim.stdout_filename
if not os.path.isdir(os.path.dirname(self.modelpath + self.sim.stdout_filename)):
os.makedirs(os.path.dirname(self.modelpath + self.sim.stdout_filename))
- #if os.name == "nt":
- # self.process = subprocess.Popen('"%s" %s 1> %s 2>&1' % (hawc2exe, htcfile, stdout), stdout=None, stderr=None, shell=True, cwd=modelpath) #, creationflags=CREATE_NO_WINDOW)
- #else:
- # self.process = subprocess.Popen('wine "%s" %s 1> %s 2>&1' % (hawc2exe, htcfile, stdout), stdout=None, stderr=None, shell=True, cwd=modelpath)
-
+
with open (os.path.join(self.modelpath, stdout), 'wb') as stdout:
if isinstance(hawc2exe, tuple):
- self.process = subprocess.Popen(list(hawc2exe + (htcfile, stdout)), stdout=stdout, stderr=STDOUT, shell=False, cwd=modelpath)
+ wine, hawc2exe = hawc2exe
+ self.process = subprocess.Popen(" ".join([wine, hawc2exe, htcfile]), stdout=stdout, stderr=STDOUT, shell=True, cwd=modelpath) #shell must be True to inwoke wine
else:
self.process = subprocess.Popen([hawc2exe, htcfile], stdout=stdout, stderr=STDOUT, shell=False, cwd=modelpath) #, creationflags=CREATE_NO_WINDOW)
self.process.communicate()
-
-# if isinstance(hawc2exe, tuple):
-# self.process = subprocess.Popen('%s "%s" %s 1> %s 2>&1' % (hawc2exe + (htcfile, stdout)), stdout=None, stderr=None, shell=False, cwd=modelpath)
-# else:
-# self.process = subprocess.Popen('"%s" %s 1> %s 2>&1' % (hawc2exe, htcfile, stdout), stdout=None, stderr=None, shell=False, cwd=modelpath) #, creationflags=CREATE_NO_WINDOW)
-# with open("cmd.txt", 'w') as fid:
-# fid.write("cd %s\n" % modelpath)
-# fid.write('"%s" %s 1> %s 2>&1' % (hawc2exe, htcfile, stdout))
-
import psutil
try:
self.sim.host.resource.process_name = psutil.Process(self.process.pid).name()
@@ -506,7 +493,7 @@ class SimulationThread(Thread):
def stop(self):
if hasattr(self, 'process'):
subprocess.Popen("TASKKILL /F /PID {pid} /T".format(pid=self.process.pid))
-
+
class PBSClusterSimulationHost(SimulationResource, SSHClient):
def __init__(self, simulation, resource, host, username, password, port=22):
@@ -649,7 +636,7 @@ class PBSClusterSimulationHost(SimulationResource, SSHClient):
#PBS -j oe
#PBS -o %s
### Maximum wallclock time format HOURS:MINUTES:SECONDS
-#PBS -l walltime=01:00:00
+#PBS -l walltime=04:00:00
###PBS -a 201547.53
#PBS -lnodes=1:ppn=1
### Queue name
@@ -661,6 +648,11 @@ cp -R . /scratch/$USER/$PBS_JOBID
### Execute commands on scratch nodes
cd /scratch/$USER/$PBS_JOBID
pwd
+export PATH=/home/python/miniconda3/bin:$PATH
+source activate wetb_py3
+WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+### modelpath: %s
+### htc: %s
echo "---------------------"
%s -c "from wetb.hawc2.cluster_simulation import ClusterSimulation;ClusterSimulation('.','%s', ('%s','%s'))"
echo "---------------------"
@@ -672,7 +664,7 @@ cd /scratch/$USER/$PBS_JOBID
echo $PBS_JOBID
cd /scratch/
### rm -r $PBS_JOBID
-exit""" % (self.simulation_id, self.stdout_filename, self.resource.python_cmd, rel_htcfilename, self.resource.wine_cmd, self.hawc2exe, cp_back)
+exit""" % (self.simulation_id, self.stdout_filename, self.modelpath, self.htcFile.filename, self.resource.python_cmd, rel_htcfilename, self.resource.wine_cmd, self.hawc2exe, cp_back)
diff --git a/wetb/prepost/Simulations.py b/wetb/prepost/Simulations.py
index a24930156d2e822d3e69b9f5757f88c8d7f58a7d..585d5af901ebe7c5681d7531fdedbfbef759df42 100755
--- a/wetb/prepost/Simulations.py
+++ b/wetb/prepost/Simulations.py
@@ -1059,6 +1059,19 @@ def post_launch(cases, save_iter=False, silent=False, suffix=None,
----------
cases : either a string (path to file) or the cases itself
+
+ save_iter : boolean, default=False
+ Set to True to save the number of iterations per time step in
+ *.iter file (in the same folder as the logfile)
+
+ path_errorlog : str, default=None
+ Root path of the error logfiles. If set to None (default), the
+ value set in the [run_dir] tag is used as the root folder of the
+ logfiles.
+
+ suffix : str, default=None
+ If not None, the suffix will be appended to file name of the error
+ log analysis file as follows: "ErrorLog_suffix.csv".
"""
# TODO: finish support for default location of the cases and file name
@@ -1443,7 +1456,7 @@ class HtcMaster(object):
for root, dirs, files in os.walk(os.path.join(plocal, path)):
for file_name in files:
src = os.path.join(root, file_name)
- dst = root.replace(os.path.abspath(plocal),
+ dst = os.path.abspath(root).replace(os.path.abspath(plocal),
os.path.abspath(prun))
if not os.path.exists(dst):
os.makedirs(dst)
@@ -1943,7 +1956,12 @@ class PBS(object):
self.wine = 'time WINEARCH=win32 WINEPREFIX=~/.wine32 wine'
else:
raise UserWarning('server support only for jess or gorm')
- self.winefix = 'WINEARCH=win32 WINEPREFIX=~/.wine32 winefix\n'
+
+ # determine at runtime if winefix has to be ran
+ self.winefix = ' _HOSTNAME_=`hostname`\n'
+ self.winefix += ' if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then \n'
+ self.winefix += ' WINEARCH=win32 WINEPREFIX=~/.wine32 winefix\n'
+ self.winefix += ' fi\n'
# the output channels comes with a price tag. Each time step
# will have a penelty depending on the number of output channels
@@ -2229,7 +2247,7 @@ class PBS(object):
self.pbs += " cp -R $PBS_O_WORKDIR/%s ./%s \n" % (fname, fgen)
# only apply the wine fix in PBS mode
- self.pbs += ' ' + self.winefix
+ self.pbs += self.winefix
# TODO: activate python env, calculate post-processing
# self.pbs += 'echo `python -c "import wetb; print(wetb.__version__)"`\n'
@@ -3559,20 +3577,41 @@ class Cases(object):
launch(self.cases, runmethod=runmethod, verbose=verbose, silent=silent,
check_log=check_log, copyback_turb=copyback_turb)
- def post_launch(self, save_iter=False, copy_pbs_failed=True, suffix=None,
+ def post_launch(self, save_iter=False, pbs_failed_path=False, suffix=None,
path_errorlog=None, silent=False):
"""
Post Launching Maintenance
check the logs files and make sure result files are present and
accounted for.
+
+ Parameters
+ ----------
+
+ save_iter : boolean, default=False
+ Set to True to save the number of iterations per time step in
+ *.iter file (in the same folder as the logfile)
+
+ pbs_failed_path : str, default=False
+ If not False, specify the path to which the *.p files of the
+ failed cases should be copied to. For example, the dlctemplate
+ will set this value to "pbs_in_fail".
+
+ path_errorlog : str, default=None
+ Root path of the error logfiles. If set to None (default), the
+ value set in the [run_dir] tag is used as the root folder of the
+ logfiles.
+
+ suffix : str, default=None
+ If not None, the suffix will be appended to file name of the error
+ log analysis file as follows: "ErrorLog_suffix.csv".
"""
# TODO: integrate global post_launch in here
self.cases_fail = post_launch(self.cases, save_iter=save_iter,
suffix=suffix, path_errorlog=path_errorlog)
- if copy_pbs_failed:
- copy_pbs_in_failedcases(self.cases_fail, path='pbs_in_fail',
+ if pbs_failed_path is not False:
+ copy_pbs_in_failedcases(self.cases_fail, path=pbs_failed_path,
silent=silent)
if self.rem_failed:
@@ -4613,6 +4652,12 @@ class Cases(object):
# ---------------------------------------------------------------------
# column definitions
# ---------------------------------------------------------------------
+ # FIXME: for backward compatibility, the column name of the unique
+ # channel name has been changed in the past....
+ if 'unique_ch_name' in dfs.columns:
+ chan_col_name = 'unique_ch_name'
+ else:
+ chan_col_name = 'channel'
# available material constants
ms, cols = [], []
for key in dfs:
@@ -4621,7 +4666,7 @@ class Cases(object):
# when multiple DLC cases are included, add extra cols to identify each
# DLC group. Make a copy, because extra_cols does not get re-initiated
# when defined as an optional keyword argument
- extra_cols_ = copy.copy(extra_cols + ['channel'])
+ extra_cols_ = copy.copy(extra_cols + [chan_col_name])
cols = copy.copy(ms)
cols.extend(extra_cols_)
# ---------------------------------------------------------------------
@@ -4632,7 +4677,7 @@ class Cases(object):
dfs = dfs.set_index('[case_id]')
# which rows to keep: a
# select for each channel all the cases
- for grname, gr in dfs.groupby(dfs.channel):
+ for grname, gr in dfs.groupby(dfs[chan_col_name]):
# if one m has any nan's, assume none of them are good and throw
# away
# if np.isnan(gr[ms[0]].values).any():
diff --git a/wetb/prepost/dlctemplate.py b/wetb/prepost/dlctemplate.py
old mode 100755
new mode 100644
index fdc21f63a0b024f5abc6ef294c35683eccef59e1..07213815eefd7877dac5d103a8ec99fcf3ad66b2
--- a/wetb/prepost/dlctemplate.py
+++ b/wetb/prepost/dlctemplate.py
@@ -266,7 +266,7 @@ def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True,
m=[1, 3, 4, 5, 6, 8, 10, 12, 14], neq=None, no_bins=46,
years=20.0, fatigue=True, A=None, AEP=False,
save_new_sigs=False, envelopeturbine=False, envelopeblade=False,
- save_iter=False):
+ save_iter=False, pbs_failed_path=False):
# =========================================================================
# check logfiles, results files, pbs output files
@@ -286,7 +286,7 @@ def post_launch(sim_id, statistics=True, rem_failed=True, check_logs=True,
cc.cases[case]['[run_dir]'] = force_dir
if check_logs:
- cc.post_launch(save_iter=save_iter)
+ cc.post_launch(save_iter=save_iter, pbs_failed_path=pbs_failed_path)
elif rem_failed:
cc.remove_failed()
@@ -360,6 +360,11 @@ if __name__ == '__main__':
dest='prep', help='create htc, pbs, files')
parser.add_argument('--check_logs', action='store_true', default=False,
dest='check_logs', help='check the log files')
+ parser.add_argument('--pbs_failed_path', default='pbs_in_fail', type=str,
+ action='store', dest='pbs_failed_path',
+ help='Copy pbs launch files of the failed cases to a '
+ 'new directory in order to prepare a re-run. Default '
+ 'value: pbs_in_failed.')
parser.add_argument('--stats', action='store_true', default=False,
dest='stats', help='calculate statistics and 1Hz '
'equivalent loads')
@@ -443,7 +448,7 @@ if __name__ == '__main__':
force_dir=P_RUN, saveinterval=2000, csv=opt.csv,
statistics=opt.stats, years=opt.years, neq=opt.neq,
fatigue=opt.fatigue, A=opt.rotarea, AEP=opt.AEP,
- no_bins=opt.no_bins, nn_blr=opt.nn_blr, nn_twt=opt.nn_twt,
+ no_bins=opt.no_bins, pbs_failed_path=opt.pbs_failed_path,
save_new_sigs=opt.save_new_sigs, save_iter=False,
envelopeturbine=opt.envelopeturbine,
envelopeblade=opt.envelopeblade)
diff --git a/wetb/prepost/misc.py b/wetb/prepost/misc.py
index f25b97897d3060b4591d7160c9c4590cc6aba107..d95b2b4fdc30686fada422ccf5930fa875e23b66 100644
--- a/wetb/prepost/misc.py
+++ b/wetb/prepost/misc.py
@@ -19,22 +19,17 @@ from future import standard_library
standard_library.install_aliases()
from builtins import object
-
-
-#print(*objects, sep=' ', end='\n', file=sys.stdout)
import os
import sys
import shutil
import unittest
import pickle
-#from xlrd import open_workbook
import numpy as np
import scipy as sp
from scipy import optimize as opt
from scipy import stats
-#import scipy.interpolate
-#import scipy.ndimage
+from scipy.interpolate import griddata as interp
from matplotlib import pyplot as plt
import pandas as pd
@@ -950,6 +945,9 @@ def histfit(hist, bin_edges, xnew):
http://nbviewer.ipython.org/url/xweb.geos.ed.ac.uk/~jsteven5/blog/
fitting_distributions_from_percentiles.ipynb
+ Calculate the CDF of given PDF, and fit a lognorm distribution onto the
+ CDF. This obviously only works if your PDF is lognorm.
+
Parameters
----------
@@ -988,6 +986,87 @@ def histfit(hist, bin_edges, xnew):
return shape_out, scale_out, pdf_fit
+def histfit_arbritrary(edges, pdf, edges_new, resolution=100):
+ """Re-bin based on the CDF of a PDF. Assume normal distribution within
+ a bin to transform the CDF to higher resolution.
+
+ Parameters
+ ----------
+
+ edges : ndarray(n+1)
+ edges of the bins, inlcuding most left and right edges.
+
+ pdf : ndarray(n)
+ probability of the bins
+
+ edges_new : ndarray(m+1)
+ edges of the new bins
+
+ resolution : int
+ resolution of the intermediate CDF used for re-fitting.
+
+
+ Returns
+ -------
+
+ centers_new : ndarray(m)
+
+ widths_new : ndarray(m)
+
+ pdf_new : ndarray(m)
+
+ """
+
+ x_hd = np.ndarray((0,))
+ cdf_hd = np.ndarray((0,))
+ binw = np.ndarray((0,))
+
+ for i in range(len(pdf)):
+ # HD grid for x
+ x_inc = np.linspace(edges[i], edges[i+1], num=resolution)
+
+ # FIXME: let the distribution in a bin be a user configurable input
+ # define a distribution within the bin: norm
+ shape = 2.5
+ scale = shape*2/10
+ x_inc = np.linspace(0, scale*10, num=resolution)
+ cdf_inc = stats.norm.cdf(x_inc, shape, scale=scale)
+
+ # scale cdf_inc and x-coordinates
+ cdf_inc_scale = pdf[i] * cdf_inc / cdf_inc[-1]
+ binw = edges[i+1] - edges[i]
+ x_inc_scale = edges[i] + (binw * x_inc / x_inc[-1])
+
+ # add to the new hd corodinates and cdf
+ x_hd = np.append(x_hd, x_inc_scale)
+ if i == 0:
+ cdf_i = 0
+ else:
+ cdf_i = cdf_hd[-1]
+ cdf_hd = np.append(cdf_hd, cdf_inc_scale + cdf_i)
+
+# plt.plot(x_inc, cdf_inc)
+# plt.plot(x_inc_scale, cdf_inc_scale)
+
+ cdf_new = interp(x_hd, cdf_hd, edges_new)
+ # last point includes everything that comes after
+ cdf_new[-1] = 1
+ pdf_new = np.diff(cdf_new)
+ widths_new = np.diff(edges_new)
+ centers_new = widths_new + edges[0]
+ # the first bin also includes everything that came before
+ pdf_new[0] += cdf_new[0]
+ pdf_new /= pdf_new.sum()
+
+# plt.plot(x_hd, cdf_hd)
+# plt.plot(edges_new, cdf_new, 'rs')
+#
+# plt.bar(edges_new[:-1], pdf_new, width=widths_new, color='b')
+# plt.bar(edges[:-1], pdf, width=np.diff(edges), color='r', alpha=0.7)
+
+ return centers_new, widths_new, pdf_new
+
+
def hist_centers2edges(centers):
"""Given the centers of bins, return its edges and bin widths.
"""
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p
index ad37d08eda8af66d8d9039f92b23411e9adafae0..27dde9c45eef7778dddad7e0b162ccc32fc2d0f4 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp10_s100.p
@@ -37,7 +37,10 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp10_s100.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/turb_s100_10ms*.bin turb/
- WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ _HOSTNAME_=`hostname`
+ if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
+ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ fi
# ------------------------------------------------------------
else
# with find+xargs we first browse to CPU folder
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p
index d4675f07d0ad5d8011fac3c5c19201705054ac8e..4d0df37617d5bf8093c7572c7ace7102da7b125c 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp11_s101.p
@@ -37,7 +37,10 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp11_s101.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/turb_s101_11ms*.bin turb/
- WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ _HOSTNAME_=`hostname`
+ if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
+ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ fi
# ------------------------------------------------------------
else
# with find+xargs we first browse to CPU folder
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p
index 8570f9cb95946522ea0f34bdfe4d445b701202e7..fdb203d5a0361e9da54c23ee3cddc1acaca6bd52 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp8_noturb.p
@@ -37,7 +37,10 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp8_noturb.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/none*.bin turb/
- WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ _HOSTNAME_=`hostname`
+ if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
+ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ fi
# ------------------------------------------------------------
else
# with find+xargs we first browse to CPU folder
diff --git a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p
index 8b7dc11bda27c4205e9eb084657ab876200fb122..84d71cd6d5066ef8a8f131e6c0e5a33f7120b6c5 100644
--- a/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p
+++ b/wetb/prepost/tests/data/demo_dlc/ref/pbs_in/dlc01_demos/dlc01_steady_wsp9_noturb.p
@@ -37,7 +37,10 @@ if [ -z ${LAUNCH_PBS_MODE+x} ] ; then
mkdir -p turb/
cp -R $PBS_O_WORKDIR/htc/dlc01_demos/dlc01_steady_wsp9_noturb.htc ./htc/dlc01_demos/
cp -R $PBS_O_WORKDIR/../turb/none*.bin turb/
- WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ _HOSTNAME_=`hostname`
+ if [[ ${_HOSTNAME_:0:1} == "j" ]] ; then
+ WINEARCH=win32 WINEPREFIX=~/.wine32 winefix
+ fi
# ------------------------------------------------------------
else
# with find+xargs we first browse to CPU folder
diff --git a/wetb/utils/caching.py b/wetb/utils/caching.py
index 84a38a69b5a171244a2aadc4519cbacb3415646a..1ca38d4bf4d77251898cfdd9921fbdb3c5828702 100644
--- a/wetb/utils/caching.py
+++ b/wetb/utils/caching.py
@@ -35,6 +35,8 @@ def set_cache_property(obj, name, get_func, set_func=None):
>>> e = Example()
>>> e.test # Call, store and return result of e.slow_function
>>> e.test # Return stored result of e.slow_function
+ >>> e._test = None # clear cache result
+ >>> e.test # Call, store and return result of e.slow_function
"""
_name = "_" + name
setattr(obj, _name, None)
diff --git a/wetb/utils/cluster_tools/ssh_client.py b/wetb/utils/cluster_tools/ssh_client.py
index 786437cf30de22e951caa201416bcbd0a30d70ef..015d75cd0def937bd5f5d214d6809882e666f690 100644
--- a/wetb/utils/cluster_tools/ssh_client.py
+++ b/wetb/utils/cluster_tools/ssh_client.py
@@ -158,11 +158,7 @@ class SSHClient(object):
cwd = os.path.join(cwd, os.path.split(filepattern)[0]).replace("\\", "/")
filepattern = os.path.split(filepattern)[1]
_, out, _ = self.execute(r'find %s -maxdepth 1 -type f -name "%s"' % (cwd, filepattern))
- files = []
- for file in out.strip().split("\n"):
- if file != "":
- files.append(file.strip())
- return files
+ return [file for file in out.strip().split("\n") if file != ""]
class SharedSSHClient(SSHClient):
diff --git a/wetb/utils/cython_compile/__init__.py b/wetb/utils/cython_compile/__init__.py
deleted file mode 100644
index 7db504d7fb31152836d8c88fcbc87882dff54f60..0000000000000000000000000000000000000000
--- a/wetb/utils/cython_compile/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from __future__ import unicode_literals
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-d = None;d = dir()
-
-from .cython_compile import cython_compile, cython_import, cython_compile_autodeclare, is_compiled
-
-__all__ = [m for m in set(dir()) - set(d)]
\ No newline at end of file
diff --git a/wetb/utils/cython_compile/cython_compile.py b/wetb/utils/cython_compile/cython_compile.py
deleted file mode 100644
index 1e17e16f68208962ccf5a20e72efe528e29c178b..0000000000000000000000000000000000000000
--- a/wetb/utils/cython_compile/cython_compile.py
+++ /dev/null
@@ -1,392 +0,0 @@
-'''
-Created on 10/07/2013
-
-@author: Mads M. Pedersen (mmpe@dtu.dk)
-
-
-Wrapper functions and decorators for compiling functions using Cython
-
-'''
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-from builtins import dict
-from io import open
-from builtins import zip
-from builtins import str
-from future import standard_library
-standard_library.install_aliases()
-import inspect
-import os
-import re
-import shutil
-import subprocess
-import sys
-import numpy as np
-import warnings
-import imp
-
-
-
-def wrap(f, autodeclare, *args, **kwargs):
- """
- Wrapper function returned by the cython_compile and cython_compile_autodeclare decorators
-
- :param f: Function to compile
- :type f: function
- :param py2pyx_func: py2pyx or py2pyx_autodeclare
- :type py2pyx_func: function
- """
-
- # Generate name: "c:\documents\project\mymodule.py" -> mymodule_myfunction
- # When called from ipython notebooks, filename is an object e.g: "<ipython-input-12-e897f9fefc0c>"
- # therefore <,>,- is removed to make it a legal python module name
- name = os.path.relpath(inspect.getabsfile(f), str(os.getcwd())).replace(".py", "")
- name = name.replace("<", "").replace(">", "").replace("-", "")
- name = "%s_%s" % (name, f.__name__)
- if name.startswith(".."):
- rel_dir = os.path.dirname(name) + os.path.sep
- sys.path.append(rel_dir)
- name = name[len(rel_dir):]
- pass
-
- module_name = name.replace(os.path.sep, ".")
-
- # import compiled module_name if exists, otherwise compile and import
- try:
- cmodule = __import__(module_name, fromlist="*")
- if cmodule.__name__ != module_name or not hasattr(cmodule, f.__name__):
- raise ImportError()
- except ImportError:
- # Generate pyrex code lines
- if autodeclare:
- pyx_lines = py2pyx_autodeclare(f, args, kwargs.copy())
- else:
- # source lines except '@cython_compile'
- source_lines = inspect.getsourcelines(f)[0][1:]
- indent = source_lines[0].index("def")
- pyx_lines = py2pyx([l[indent:] for l in source_lines])
-
- # Write pyrex code lines to .pyx file
- pyx_filename = name.replace("\\", "/") + ".pyx"
- with open(pyx_filename, 'w', encoding='utf-8') as fid:
- fid.writelines(pyx_lines)
-
- # compile, import compiled module_name and delete temporary files
- cmodule = compile_and_cleanup(module_name, pyx_filename, module_name)
- try:
- cf = getattr(cmodule, f.__name__)
- if kwargs == {}:
- return cf(*args)
- else:
- return cf(*args, **kwargs)
- except AttributeError:
- warnings.warn(
- "Compilation or import of %s failed. Python function used instead" %
- f)
- return f(*args, **kwargs)
-
-
-def cython_compile(f):
- """Decorator for compilation, import and execution of the function, f.
- Variables can be declared using Pure or cdef syntax, see module description
- Example:
- @cython_compile
- def my_func(p):
- pass
- """
- w = lambda *args, **kwargs: wrap(f, False, *args, **kwargs)
- w.__name__ = f.__name__
- return w
-
-
-def cython_compile_autodeclare(f):
- """Decorator for autodeclaring, compilation, import and execution of the function, f.
- Declared variables using cdef syntax overrides autodeclaration, see module description
- Example:
- @cython_compile_autocompile
- def my_func(p):
- pass
- """
- w = lambda *args, **kwargs: wrap(f, True, *args, **kwargs)
- w.__name__ = f.__name__
- return w
-
-
-def cython_import(import_module_name, compiler=None):
- """Compiles and imports a module. Use it similar to the normal import
- Example (import my_func from my_module):
-
- from wetb.utils.cython_compile import cython_import
- cython_import('my_module')
- import my_module # import must be after cython_import statement
- my_module.my_func()
-
- """
- module_name = import_module_name.split(".")[-1]
- globals()[module_name] = __import__(import_module_name, fromlist=module_name)
-
- pyd_module = eval(module_name)
- if not is_compiled(pyd_module):
-
- # Read py-module
- file_name = import_module_name.replace(".", "/") + ".py"
- for p in ["."] + sys.path:
- if os.path.isfile(os.path.join(p, file_name)):
- file_path = os.path.join(p, file_name).replace("\\", "/")
- break
- else:
- raise IOError("Module %s cannot be found" % file_name)
-
- fid = open(file_path)
- pylines = fid.readlines()
- fid.close()
-
- # write pyrex file
- pyx_filename = file_path.replace('.py', '.pyx')
- fid = open(pyx_filename, 'w')
- pyxlines = py2pyx(pylines)
- fid.writelines(pyxlines)
- fid.close()
-
- # compile, import compiled module and delete temporary files
- module_relname = eval(module_name).__package__ + "." + module_name #"os.path.relpath(eval(module_name).__file__, str(os.getcwd())).replace(os.path.sep, ".")[:-3]
- return compile_and_cleanup(import_module_name, pyx_filename, module_relname, compiler)
- return eval(module_name)
-
-
-def compile_and_cleanup(module, pyx_filename, module_relname, compiler=None):
- """compile, import compiled module and delete temporary files"""
-
- # Generate setup.py script
- fid = open('setup.py', 'w')
- setup_str = """from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Distutils import build_ext
-import numpy
-
-ext_modules = [Extension("%s", ["%s"], include_dirs = [numpy.get_include()])]
-
-setup(
-name = 'name',
-cmdclass = {'build_ext': build_ext},
-ext_modules = ext_modules
-)""" % (module_relname, pyx_filename)
- fid.write(setup_str)
- fid.close()
- if hasattr(sys, 'frozen'):
- raise Exception ("Trying to compile %s in frozen application" % str(module))
- # create compile command
- import platform
- if compiler is not None:
- compiler_str = "--compiler=%s" % compiler
- elif platform.architecture()[0] == "32bit" and os.name == 'nt' and "mingw" in os.environ['path'].lower():
- compiler_str = "--compiler=mingw32"
- else:
- compiler_str = ""
-
-
-
- bin_python = sys.executable
- cmd = "%s setup.py build_ext --inplace %s" % (bin_python, compiler_str)
-
- # compile
- print ("compiling %s: %s" % (module, cmd))
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
- (out, err) = proc.communicate()
-
- # Reload and check that module is compiled
- try:
- try:
- del sys.modules[module]
- except KeyError:
- pass
- cmodule = __import__(module, fromlist=module.split(".")[-1])
- imp.reload(cmodule)
-
- except ImportError:
- cmodule = None
-
- if cmodule is None or is_compiled(cmodule) == False:
- line = '\n' + '=' * 79 + '\n'
- sys.stderr.write("%s was not compiled correctly. It may result in slower execution" % module)
- sys.stderr.write('%sstdout:\n%s' % (line, out.decode()))
- sys.stderr.write('%sstderr:%s%s' % (line, line, err))
-
- else:
- print ("Compiling succeeded")
- for f in ['setup.py', pyx_filename, pyx_filename.replace(".pyx", '.c')]:
- if os.path.isfile(f):
- os.remove(f)
-
- # Clean up. Remove temporary files and folders
- if os.path.isdir("build"):
- try:
- shutil.rmtree("build")
- except:
- pass
-
- return cmodule
-
-
-def py2pyx_autodeclare(f, args, kwargs):
- """Generate pyrex code of function f and its input arguments
- This function invokes py2pyx and extends with autodeclarations:
- - arguments: input arguments are declared based their values in args and kwargs
- - cdef: Variables declared by cdef, overrides autodeclaration, e.g:
- "def func(a): #cpdef func(int a):" -> "cpdef func(int a):" (independent of type of a)
- "#cdef int a" - "cdef int a" (independent of type of a)
- - assignment: Variables assigned in function, e.g.
- "a = xxx"
- are declared based on the type of eval(xxx)
- - in: variables returned by iterators, e.g:
- "for a in range(5)",
- "for a,b in [(1,.2)]:"
- are declared base on the type of the first element in eval(iterator), e.g. "type(eval(range(5)[0]))"
- """
-
- arg_names, _, _, defaults = inspect.getargspec(f)
-
- # update kwargs with defaults
- if defaults:
- for k, v in zip(arg_names[::-1], defaults[::-1]):
- if k not in kwargs:
- kwargs[k] = v
- kwargs.update(dict(zip(arg_names, args)))
-
- # get pyx code lines using py2pyx
- lines = inspect.getsourcelines(f)[0]
- if lines[0].strip() == "@cython_compile_autodeclare":
- lines = lines[1:]
- lines = py2pyx(lines)
-
- # prepare regular expressions
- var_name = "(?:_*[a-zA-Z][a-zA-Z0-9_]*)" # optional "_" + one alpha + [0..n] x alphanum. "?:" = no group
- reg_args = re.compile("[ \t]*def +(?:%s) *\(([^:]*)\) *:" % (var_name))
- reg_import = re.compile("[ \t]*c?import *(%s)(?: +as +(%s))?" % (var_name, var_name))
- reg_cdef = re.compile("[ \t]*cdef +(?:(?:signed|unsigned|int|long|float|double|np\.ndarray\[.*\]) *)*(%s)" % var_name)
- reg_assign = re.compile('[ \t]*(%s) *=(.*)' % var_name)
- reg_in = re.compile('[ \t]*for +(%s(?:, *%s)*) +in +(.*):' % (var_name, var_name))
-
-
- def declare_str(var_name, var_value):
- """Generate declaration string '<type(var_value)> <var_name>' e.g:
- declare_str('a',1) -> "long a"
- """
- if isinstance(var_value, (int)):
- return "long long %s" % var_name
- if isinstance(var_value, float):
- return "double %s" % var_name
- elif isinstance(var_value, np.ndarray):
- return "np.ndarray[np.%s_t,ndim=%d] %s" % (var_value.dtype, len(var_value.shape), var_name)
- else:
- raise NotImplementedError(type(var_value))
-
- defs = {} # dict for known local variables
- def_line = None # line nr of "def func():". Autodeclaration of local field inserted below this line
- for i, line in enumerate(lines):
- if def_line is None and 'def' in line:
- # first line containing "def" = function declaration line
- def_line = i
- match = reg_args.match(line)
- if match is not None:
- args = match.group(1).strip() # line=" def func(xxx):#comment" -> args='xxx'
- arg_strs = []
- if args != "":
- for arg in args.split(','):
- arg_name = arg.split('=')[0].strip()
- arg_value = kwargs[arg_name]
- try:
- arg_strs.append(arg.strip().replace(arg_name, declare_str(arg_name, arg_value), 1))
- except NotImplementedError:
- arg_strs.append(arg)
- # replace function declaration line
- lines[i] = '%scpdef %s(%s):\n' % (" " * line.index('def'), f.__name__, ", ".join(arg_strs))
- else:
- #Import: import x as y
- match = reg_import.match(line)
- if match is not None:
- # add imported moduled to kwargs -> enable evaluation of variables
- import_module = match.group(1)
- if match.group(2) is None:
- kwargs[import_module] = __import__(import_module)
- else:
- # import xxx as y
- kwargs[match.group(2)] = __import__(import_module)
- continue
- #Cdef: cdef int x
- match = reg_cdef.match(line)
- if match is not None:
- # line contains a 'cdef' declaration.
- # Add to defs to avoid redeclaration
- var_name = match.group(1)
- if var_name not in defs:
- defs[var_name] = None
- continue
- #Assignment: x = y
- match = reg_assign.match(line)
- if match is not None:
- # line contains an assignment, e.g. a = xxx.
- # Try to evaluate xxx and declare a as type of eval(xxx)
- try:
- var_name = match.group(1)
- if var_name not in defs:
- var_value = eval(match.group(2), globals(), kwargs)
- defs[var_name] = declare_str(
- var_name.strip(), var_value)
- kwargs[var_name] = var_value
- except NotImplementedError:
- pass
- continue
- #In: For x in y
- match = reg_in.match(line)
- if match is not None:
- # line contains 'for xxx in yyy:'.
- # Try to evaluate yyy and declare xxx as type of first element
- # of eval(yyy)
- var_names = [v.strip() for v in match.group(1).split(",")]
- var_values = eval(match.group(2), globals(), kwargs)[0]
- if not isinstance(var_values, (list, tuple)):
- var_values = (var_values,)
- for var_name, var_value in zip(var_names, var_values):
- try:
- if var_name not in defs:
- defs[var_name] = declare_str(var_name, var_value)
- kwargs[var_name] = var_value
- except NotImplementedError:
- pass
- indent = lines[def_line + 1].replace(lines[def_line + 1].lstrip(), "")
-
- # Insert declaration of local fields ordered by name just below function
- # declaration
- for key in sorted(defs.keys(), reverse=True):
- if defs[key] is not None:
- lines.insert(def_line + 1, "%scdef %s\n" % (indent, defs[key]))
-
- return lines
-
-
-def py2pyx(pylines):
- """Generate pyrex code lines from python code lines
- - Adds import of cython and numpy
- - searches every line for "#c". If found text before "#c" is replaced with text after "#c", e.g:
- "def func(a): #cpdef func(int a):" -> "cpdef func(int a):
- "#cdef int b" -> "cdef int b"
- """
- pyxlines = ['import cython\n', 'import numpy as np\n', 'cimport numpy as np\n']
- for l in pylines:
-
- if "#c" in l:
- indent = l[:len(l) - len(l.lstrip())]
- cdef = l[l.index("#c") + 1:]
- l = indent + cdef
- if "__future__" in l:
- pyxlines.insert(0, l) # from __future__ import ... must be first
- else:
- pyxlines.append(l)
- return pyxlines
-
-
-def is_compiled(module):
- return module.__file__.lower()[-4:] == ".pyd" or module.__file__.lower()[-3:] == ".so"
diff --git a/wetb/utils/cython_compile/examples/__init__.py b/wetb/utils/cython_compile/examples/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/wetb/utils/cython_compile/examples/cycheck.py b/wetb/utils/cython_compile/examples/cycheck.py
deleted file mode 100644
index 602cbf3521261a1565a51dc443890686665cb60c..0000000000000000000000000000000000000000
--- a/wetb/utils/cython_compile/examples/cycheck.py
+++ /dev/null
@@ -1,41 +0,0 @@
-'''
-Created on 29/03/2013
-
-@author: Mads
-'''
-from __future__ import unicode_literals
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-from builtins import range
-from builtins import int
-from future import standard_library
-standard_library.install_aliases()
-import cython
-import math
-
-
-def cycheck(p):
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
-
-@cython.ccall
-@cython.locals(y=cython.int, p=cython.ulonglong)
-def cycheck_pure(p):
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
-
-
-def cycheck_cdef(p): #cpdef cycheck_cdef(unsigned long long p):
- #cdef int y
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
diff --git a/wetb/utils/cython_compile/examples/examples.py b/wetb/utils/cython_compile/examples/examples.py
deleted file mode 100644
index 14f794fe31c4866f8f8597e2ace752e2322c8c74..0000000000000000000000000000000000000000
--- a/wetb/utils/cython_compile/examples/examples.py
+++ /dev/null
@@ -1,62 +0,0 @@
-'''
-Created on 11/07/2013
-
-@author: Mads M. Pedersen (mmpe@dtu.dk)
-'''
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-from builtins import range
-from builtins import int
-from future import standard_library
-standard_library.install_aliases()
-import math
-
-from wetb.utils.cython_compile.cython_compile import cython_compile, \
- cython_compile_autodeclare, cython_import
-from wetb.utils.cython_compile.examples import cycheck
-
-
-def pycheck(p):
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
-
-
-@cython_compile
-def cycheck_compile(p):
- import math
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
-
-
-@cython_compile_autodeclare
-def cycheck_compile_autodeclare(p):
- import math
- for i in range(10):
- for y in range(2, int(math.sqrt(p)) + 1):
- if p % y == 0:
- return False
- return True
-
-if __name__ == "__main__":
- p = 17
-
- print (pycheck(p))
-
- cython_import('cycheck')
- print (cycheck.cycheck(p))
- print (cycheck.cycheck_pure(p))
- print (cycheck.cycheck_cdef(p))
-
- print (cycheck_compile(p))
-
- print (cycheck_compile_autodeclare(p))
-
-